Began implementing fully isomorphic macros

Like Rust's Proc macros. Now we have preprocessor recursion to worry about. I also made a cool macro for enums

.zed/settings.json

@@ -1,7 +1,8 @@
 {
   "languages": {
     "Rust": {
-      "language_servers": ["rust-analyzer", "..."]
+      "language_servers": ["rust-analyzer", "..."],
+      "formatter": "language_server"
     }
   },
   "wrap_guides": [100],
@@ -11,8 +12,11 @@
         "path": "C:\\Users\\z004yk5r\\.cargo\\bin\\rust-analyzer.exe"
       },
       "initialization_options": {
-        "checkOnSave": {
+        "check": {
           "command": "clippy"
+        },
+        "rustfmt": {
+          "extraArgs": ["+nightly"]
         }
       }
     }

Cargo.lock

@@ -477,9 +477,9 @@ checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91"
 
 [[package]]
 name = "num-traits"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
 dependencies = [
  "autocfg",
 ]
@@ -515,6 +515,7 @@ dependencies = [
 name = "orchid-api-traits"
 version = "0.1.0"
 dependencies = [
+ "itertools",
  "never",
  "ordered-float",
 ]
@@ -529,6 +530,7 @@ dependencies = [
  "itertools",
  "lazy_static",
  "never",
+ "num-traits",
  "orchid-api",
  "orchid-api-derive",
  "orchid-api-traits",
@@ -571,12 +573,14 @@ dependencies = [
  "itertools",
  "lazy_static",
  "never",
+ "num-traits",
  "orchid-api",
  "orchid-api-traits",
  "orchid-base",
  "ordered-float",
  "paste",
  "substack",
+ "trait-set",
 ]
 
 [[package]]
@@ -925,9 +929,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "substack"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffccc3d80f0a489de67aa74ff31ab852abb973e1c6dacf3704889e00ca544e7f"
+checksum = "26ce98c74d8476dd7b8515495625bc1bd4449b50f4926ac030964976e035ed53"
 
 [[package]]
 name = "syn"

examples/hello-world/main.orc

@@ -1,35 +1,2 @@
-const main := println "Hello World!" exit_status::success
+const user := "dave"
-
+const main := println "Hello $user!" exit_status::success
-macro (
-	rule match ...$expr { ...$body } => '(
-		fn::pass (...$expr) \match::value. ...$(
-			fn::pass (quote::split body ';) \cases.
-			fn::pass (list::map cases \x. (
-				fn::pass (quote::split_once x '=>) \pair.
-				tuple::destr pair 2 \req. \handler.
-				fn::pass (macro::run '(match::request (...$key))) \match_res.
-				quote::match '(match::response $decoder (...$bindings)) match_res \match_res_match.
-				fn::pass (option::expect match_res_match "Invalid pattern ${key}") \res_parts.
-				fn::pass (map::get_unwrap res_parts "decoder") \decoder.
-				fn::pass (map::get_unwrap res_parts "bindings") \bindings.
-				fn::pass (quote::to_list bindings) \binding_names.
-				fn::pass (list::rfold handler \tail. \name. '( \ $name . $tail )) \success.
-				'( $decoder $success )
-			)) \case_fns.
-			list::append case_fns '( panic "No cases match" )
-		)
-	)
-)
-
---[
-	Conceptually, all matches are compared.
-		1. Within a macro, the top rule wins
-		2. If two winning matches are not within the same macro,
-			the one that matches the outermost, first token wins,
-			including tokens that are immediately captured, such that a rule starting with .. is
-			beaten by the same rule parenthesized and any rule starting with a scalar is beaten
-			by the same rule prefixed with a vectorial
-		3. If two winning matches start with the same token, an ambiguity error is raised
-
-	
-]--

notes/new_macro_model.md

@@ -0,0 +1,45 @@
+# Code sample for the new macro system
+
+```
+macro (
+	rule match ...$expr { ...$body } => \recurse. '(
+		fn::pass (...$expr) \match::value. ...$(
+			fn::pass (quote::split body ';) \cases.
+			fn::pass (list::map cases \x. (
+				fn::pass (quote::split_once x '=>) \pair.
+				tuple::destr pair 2 \req. \handler.
+				fn::pass (recurse '(match::request (...$key))) \match_res.
+				quote::match '(match::response $decoder (...$bindings)) match_res \match_res_match.
+				fn::pass (option::expect match_res_match "Invalid pattern ${key}") \res_parts.
+				fn::pass (map::get_unwrap res_parts "decoder") \decoder.
+				fn::pass (map::get_unwrap res_parts "bindings") \bindings.
+				fn::pass (quote::to_list bindings) \binding_names.
+				fn::pass (list::rfold handler \tail. \name. '( \ $name . $tail )) \success.
+				'( $decoder $success )
+			)) \case_fns.
+			list::append case_fns '( panic "No cases match" )
+		)
+	)
+)
+
+--[
+	Macros are run from the top down.
+	For every token
+		1. If it's a name token, test all macros starting with that name
+		2. If none match and this is the first token in a list, test all macros starting with vectorials
+	Test all in a set of macros
+		1. Take the first rule that matches in each block
+		2. If there are multiple matches across blocks, raise an ambiguity error
+		3. If the single match is in the recursion stack, raise a recursion error
+		4. Add the matching rule to the recursion stack, then execute the body.
+]--
+
+--[
+	1. Macro patterns are held in the host, they don't contain atoms, and atoms are never considered equal, so the matcher doesn't have to call an extension.
+	2. The body of macros may be defined in Rust. If it isn't, the entire interpreter will recurse on the macro to calculate the output.
+]--
+
+--[
+	1. if the rule body uses the same macro, fail with the rule
+	2. if the rule explicitly recursively invokes the same macro, fail with the first match
+]--

orchid-api-traits/Cargo.toml

@@ -6,5 +6,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+itertools = "0.13.0"
 never = "0.1.0"
 ordered-float = "4.2"

orchid-api-traits/src/api_conv.rs

@@ -0,0 +1,26 @@
+pub trait ApiEquiv {
+  type Api;
+}
+
+pub trait ToApi: Sized + ApiEquiv {
+  type Ctx;
+  fn to_api(&self, ctx: &mut Self::Ctx) -> Self::Api;
+  fn into_api(self, ctx: &mut Self::Ctx) -> Self::Api { self.to_api(ctx) }
+}
+
+pub trait FromApi: ApiEquiv {
+  type Ctx;
+  fn from_api(api: &Self::Api, ctx: &mut Self::Ctx) -> Self;
+}
+
+/// This is the weakest kind of conversion possible;
+/// By holding a reference to the source type, you can provide a reference to the target type.
+/// Unlike Into, the target type may hold references into the source,
+/// but unlike AsRef, it doesn't have to be fully contained in the source.
+/// The resulting object is stackbound so its utility is very limited.
+pub trait ProjectionMut<T> {
+  fn with_built<R>(&mut self, cb: impl FnOnce(&mut T) -> R) -> R;
+}
+impl<T> ProjectionMut<T> for T {
+  fn with_built<R>(&mut self, cb: impl FnOnce(&mut T) -> R) -> R { cb(self) }
+}

orchid-api-traits/src/helpers.rs

@@ -1,5 +1,7 @@
 use std::io::{Read, Write};
 
+use itertools::{Chunk, Itertools};
+
 use crate::Encode;
 
 pub fn encode_enum<W: Write + ?Sized>(write: &mut W, id: u8, f: impl FnOnce(&mut W)) {
@@ -11,10 +13,20 @@ pub fn write_exact<W: Write + ?Sized>(write: &mut W, bytes: &'static [u8]) {
   write.write_all(bytes).expect("Failed to write exact bytes")
 }
 
+pub fn print_bytes(b: &[u8]) -> String {
+  (b.iter().map(|b| format!("{b:02x}")))
+    .chunks(4)
+    .into_iter()
+    .map(|mut c: Chunk<_>| c.join(" "))
+    .join("  ")
+}
+
 pub fn read_exact<R: Read + ?Sized>(read: &mut R, bytes: &'static [u8]) {
   let mut data = vec![0u8; bytes.len()];
   read.read_exact(&mut data).expect("Failed to read bytes");
-  assert_eq!(&data, bytes, "Wrong bytes")
+  if data != bytes {
+    panic!("Wrong bytes!\nExpected: {}\nFound: {}", print_bytes(bytes), print_bytes(&data));
+  }
 }
 
 pub fn enc_vec(enc: &impl Encode) -> Vec<u8> {

orchid-api-traits/src/lib.rs

@@ -2,8 +2,10 @@ mod coding;
 mod helpers;
 mod hierarchy;
 mod relations;
+mod api_conv;
 
-pub use coding::{Coding, Decode, Encode};
+pub use coding::*;
-pub use helpers::{enc_vec, encode_enum, read_exact, write_exact};
+pub use helpers::*;
-pub use hierarchy::{Extends, InHierarchy, TLBool, TLFalse, TLTrue, UnderRoot};
+pub use hierarchy::*;
-pub use relations::{Channel, MsgSet, Request};
+pub use relations::*;
+pub use api_conv::*;

orchid-api-traits/src/relations.rs

@@ -3,7 +3,11 @@ use crate::helpers::enc_vec;
 
 pub trait Request: Coding + Sized + Send + 'static {
   type Response: Coding + Send + 'static;
-  fn respond(&self, rep: Self::Response) -> Vec<u8> { enc_vec(&rep) }
+}
+
+pub fn respond<R: Request>(_: &R, rep: R::Response) -> Vec<u8> { enc_vec(&rep) }
+pub fn respond_with<R: Request>(r: &R, f: impl FnOnce(&R) -> R::Response) -> Vec<u8> {
+  respond(r, f(r))
 }
 
 pub trait Channel: 'static {

orchid-api/src/atom.rs

@@ -3,10 +3,7 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::OrcResult;
+use crate::{ExprTicket, Expression, ExtHostReq, HostExtNotif, HostExtReq, OrcResult, SysId, TStrv};
-use crate::expr::{Expr, ExprTicket};
-use crate::proto::{ExtHostReq, HostExtNotif, HostExtReq};
-use crate::system::SysId;
 
 pub type AtomData = Vec<u8>;
 
@@ -53,7 +50,7 @@ pub struct Atom {
 #[extends(AtomReq, HostExtReq)]
 pub struct CallRef(pub Atom, pub ExprTicket);
 impl Request for CallRef {
-  type Response = Expr;
+  type Response = Expression;
 }
 
 /// Attempt to apply an atom as a function, consuming the atom and enabling the
@@ -63,7 +60,7 @@ impl Request for CallRef {
 #[extends(AtomReq, HostExtReq)]
 pub struct FinalCall(pub Atom, pub ExprTicket);
 impl Request for FinalCall {
-  type Response = Expr;
+  type Response = Expression;
 }
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
@@ -80,34 +77,24 @@ impl Request for DeserAtom {
   type Response = Atom;
 }
 
-/// Determine whether two atoms are identical for the purposes of macro
-/// application. If a given atom is never generated by macros or this relation
-/// is difficult to define, the module can return false
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
-#[extends(AtomReq, HostExtReq)]
-pub struct AtomSame(pub Atom, pub Atom);
-impl Request for AtomSame {
-  type Response = bool;
-}
-
 /// A request blindly routed to the system that provides an atom.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
-pub struct Fwded(pub Atom, pub Vec<u8>);
+pub struct Fwded(pub Atom, pub TStrv, pub Vec<u8>);
 impl Request for Fwded {
-  type Response = Vec<u8>;
+  type Response = Option<Vec<u8>>;
 }
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
-pub struct Fwd(pub Atom, pub Vec<u8>);
+pub struct Fwd(pub Atom, pub TStrv, pub Vec<u8>);
 impl Request for Fwd {
-  type Response = Vec<u8>;
+  type Response = Option<Vec<u8>>;
 }
 
 #[derive(Clone, Debug, Coding)]
 pub enum NextStep {
-  Continue(Expr),
+  Continue(Expression),
   Halt,
 }
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
@@ -138,7 +125,6 @@ impl Request for AtomPrint {
 pub enum AtomReq {
   CallRef(CallRef),
   FinalCall(FinalCall),
-  AtomSame(AtomSame),
   Fwded(Fwded),
   Command(Command),
   AtomPrint(AtomPrint),
@@ -149,8 +135,7 @@ impl AtomReq {
   /// subclass have at least one atom argument.
   pub fn get_atom(&self) -> &Atom {
     match self {
-      Self::AtomSame(AtomSame(a, ..))
+      Self::CallRef(CallRef(a, ..))
-      | Self::CallRef(CallRef(a, ..))
       | Self::Command(Command(a))
       | Self::FinalCall(FinalCall(a, ..))
       | Self::Fwded(Fwded(a, ..))

orchid-api/src/error.rs

@@ -3,8 +3,7 @@ use std::sync::Arc;
 
 use orchid_api_derive::Coding;
 
-use crate::interner::TStr;
+use crate::{Location, TStr};
-use crate::location::Location;
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ErrId(pub NonZeroU16);

orchid-api/src/expr.rs

@@ -3,12 +3,7 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::atom::Atom;
+use crate::{Atom, ExtHostNotif, ExtHostReq, Location, OrcError, SysId, TStrv};
-use crate::error::OrcError;
-use crate::interner::TStrv;
-use crate::location::Location;
-use crate::proto::{ExtHostNotif, ExtHostReq};
-use crate::system::SysId;
 
 /// An arbitrary ID associated with an expression on the host side. Incoming
 /// tickets always come with some lifetime guarantee, which can be extended with
@@ -57,11 +52,11 @@ pub struct Move {
 /// [crate::atom::Call] or [crate::atom::CallRef], or a constant in the
 /// [crate::tree::Tree].
 #[derive(Clone, Debug, Coding)]
-pub enum Clause {
+pub enum ExpressionKind {
   /// Apply the lhs as a function to the rhs
-  Call(Box<Expr>, Box<Expr>),
+  Call(Box<Expression>, Box<Expression>),
   /// Lambda function. The number operates as an argument name
-  Lambda(u64, Box<Expr>),
+  Lambda(u64, Box<Expression>),
   /// Binds the argument passed to the lambda with the same ID in the same
   /// template
   Arg(u64),
@@ -70,16 +65,12 @@ pub enum Clause {
   Slot(ExprTicket),
   /// The lhs must be fully processed before the rhs can be processed.
   /// Equivalent to Haskell's function of the same name
-  Seq(Box<Expr>, Box<Expr>),
+  Seq(Box<Expression>, Box<Expression>),
   /// Insert a new atom in the tree. When the clause is used in the const tree,
   /// the atom must be trivial. This is always a newly constructed atom, if you
   /// want to reference an existing atom, use the corresponding [ExprTicket].
   /// Because the atom is newly constructed, it also must belong to this system.
   NewAtom(Atom),
-  /// An atom, specifically an atom that already exists. This form is only ever
-  /// returned from [Inspect], and it's borrowed from the expression being
-  /// inspected.
-  Atom(ExprTicket, Atom),
   /// A reference to a constant
   Const(TStrv),
   /// A static runtime error.
@@ -87,22 +78,35 @@ pub enum Clause {
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct Expr {
+pub struct Expression {
-  pub clause: Clause,
+  pub kind: ExpressionKind,
   pub location: Location,
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct Details {
+pub enum InspectedKind {
-  pub expr: Expr,
+  Atom(Atom),
+  Bottom(Vec<OrcError>),
+  Opaque,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Inspected {
+  pub kind: InspectedKind,
+  pub location: Location,
   pub refcount: u32,
 }
 
+/// Obtain information about an expression. Used to act upon arguments by
+/// resolving shallowly and operating on the atom, but also usable for
+/// reflection
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExprReq, ExtHostReq)]
-pub struct Inspect(pub ExprTicket);
+pub struct Inspect {
+  pub target: ExprTicket,
+}
 impl Request for Inspect {
-  type Response = Details;
+  type Response = Inspected;
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]

orchid-api/src/interner.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::proto::{ExtHostReq, HostExtReq};
+use crate::{ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.

orchid-api/src/lexer.rs

@@ -0,0 +1,46 @@
+use std::ops::RangeInclusive;
+
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TokenTree, TreeTicket};
+
+/// - All ranges contain at least one character
+/// - All ranges are in increasing characeter order
+/// - There are excluded characters between each pair of neighboring ranges
+#[derive(Clone, Debug, Coding)]
+pub struct CharFilter(pub Vec<RangeInclusive<char>>);
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct LexExpr {
+  pub sys: SysId,
+  pub id: ParsId,
+  pub text: TStr,
+  pub pos: u32,
+}
+impl Request for LexExpr {
+  type Response = Option<OrcResult<LexedExpr>>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct LexedExpr {
+  pub pos: u32,
+  pub expr: TokenTree,
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct SubLex {
+  pub id: ParsId,
+  pub pos: u32,
+}
+impl Request for SubLex {
+  type Response = Option<SubLexed>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct SubLexed {
+  pub pos: u32,
+  pub ticket: TreeTicket,
+}

orchid-api/src/lib.rs

@@ -1,35 +1,26 @@
+mod lexer;
+pub use lexer::*;
+mod macros;
+pub use macros::*;
 mod atom;
-pub use atom::{
+pub use atom::*;
-  Atom, AtomData, AtomDrop, AtomId, AtomPrint, AtomReq, AtomSame, CallRef, Command, DeserAtom,
-  FinalCall, Fwd, Fwded, LocalAtom, NextStep, SerializeAtom,
-};
 mod error;
-pub use error::{ErrId, ErrLocation, OrcError, OrcResult};
+pub use error::*;
 mod expr;
-pub use expr::{
+pub use expr::*;
-  Acquire, Clause, Details, Expr, ExprNotif, ExprReq, ExprTicket, Inspect, Move, Release,
-};
 mod interner;
-pub use interner::{
+pub use interner::*;
-  ExternStr, ExternStrv, IntReq, InternStr, InternStrv, Retained, Sweep, TStr, TStrv,
-};
 mod location;
-pub use location::{CodeGenInfo, Location, SourceRange};
+pub use location::*;
 mod logging;
-pub use logging::{Log, LogStrategy};
+pub use logging::*;
 mod parser;
-pub use parser::{CharFilter, LexExpr, LexedExpr, ParsId, ParseLine, ParserReq, SubLex, SubLexed};
+pub use parser::*;
 mod proto;
-pub use proto::{
+pub use proto::*;
-  ExtHostChannel, ExtHostNotif, ExtHostReq, ExtMsgSet, ExtensionHeader, HostExtChannel,
-  HostExtNotif, HostExtReq, HostHeader, HostMsgSet, Ping,
-};
 mod system;
-pub use system::{NewSystem, SysDeclId, SysId, SysReq, SystemDecl, SystemDrop, SystemInst};
+pub use system::*;
 mod tree;
-pub use tree::{
+pub use tree::*;
-  CompName, GetMember, Item, ItemKind, Member, MemberKind, Module, Paren, Token, TokenTree, TreeId,
-  TreeTicket,
-};
 mod vfs;
-pub use vfs::{EagerVfs, GetVfs, Loaded, VfsId, VfsRead, VfsReq};
+pub use vfs::*;

orchid-api/src/location.rs

@@ -2,13 +2,17 @@ use std::ops::Range;
 
 use orchid_api_derive::Coding;
 
-use crate::interner::TStrv;
+use crate::{TStr, TStrv};
 
 #[derive(Clone, Debug, Coding)]
 pub enum Location {
+  /// Location inaccessible. Locations are always debugging aids and never
+  /// mandatory.
   None,
+  /// Associated with a slot when wrapped in an expression.
+  SlotTarget,
   /// Used in functions to denote the generated code that carries on the
-  /// location of the call. Not allowed in the const tree.
+  /// location of the call.
   Inherit,
   Gen(CodeGenInfo),
   /// Range and file
@@ -26,5 +30,5 @@ pub struct SourceRange {
 #[derive(Clone, Debug, Coding)]
 pub struct CodeGenInfo {
   pub generator: TStrv,
-  pub details: String,
+  pub details: TStr,
 }

orchid-api/src/logging.rs

@@ -1,6 +1,6 @@
 use orchid_api_derive::{Coding, Hierarchy};
 
-use crate::proto::ExtHostNotif;
+use crate::ExtHostNotif;
 
 #[derive(Clone, Debug, Coding)]
 pub enum LogStrategy {

orchid-api/src/macros.rs

@@ -0,0 +1,82 @@
+use std::collections::HashMap;
+use std::num::NonZeroU64;
+
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+use ordered_float::NotNan;
+
+use crate::{Comment, ExtHostReq, HostExtReq, Location, OrcResult, Paren, ParsId, SysId, TStr, TStrv};
+
+#[derive(Clone, Debug, Coding)]
+pub struct MacroTreeId(NonZeroU64);
+
+#[derive(Clone, Debug, Coding)]
+pub struct MacroTree {
+  pub location: Location,
+  pub token: MacroToken,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum MacroToken {
+  S(Paren, Vec<MacroTree>),
+  Name(TStrv),
+  Slot(MacroTreeId),
+  Lambda(Vec<MacroTree>, Vec<MacroTree>),
+  Ph(Placeholder),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct MacroBlock {
+  pub priority: Option<NotNan<f64>>,
+  pub rules: Vec<MacroRule>,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct MacroRule {
+  pub location: Location,
+  pub comments: Vec<Comment>,
+  pub pattern: Vec<MacroTree>,
+  pub id: MacroId,
+}
+
+/// A specific macro rule with a specific pattern across invocations
+#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, Hash)]
+pub struct MacroId(pub NonZeroU64);
+
+/// After a pattern matches, this call executes the body of the macro. This request returns None
+/// if an inner nested request raised an exception
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct ApplyMacro {
+  pub sys: SysId,
+  pub id: MacroId,
+  /// Recursion token
+  pub run_id: ParsId,
+  /// Must contain exactly the keys that were specified as placeholders in the pattern
+  pub params: HashMap<TStr, Vec<MacroTree>>,
+}
+impl Request for ApplyMacro {
+  type Response = Option<OrcResult<Vec<MacroTree>>>;
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct RunMacros {
+  pub run_id: ParsId,
+  pub query: Vec<MacroTree>,
+}
+impl Request for RunMacros {
+  type Response = Option<Vec<MacroTree>>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Placeholder {
+  pub name: TStr,
+  pub kind: PhKind,
+}
+
+#[derive(Clone, Copy, Debug, Coding)]
+pub enum PhKind {
+  Scalar,
+  Vector { priority: u8, at_least_one: bool },
+}

orchid-api/src/parser.rs

@@ -1,70 +1,19 @@
 use std::num::NonZeroU64;
-use std::ops::RangeInclusive;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::OrcResult;
+use crate::{Comment, HostExtReq, OrcResult, SysId, TokenTree};
-use crate::interner::TStr;
-use crate::proto::{ExtHostReq, HostExtReq};
-use crate::system::SysId;
-use crate::tree::{TokenTree, TreeTicket};
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ParsId(pub NonZeroU64);
 
-/// - All ranges contain at least one character
-/// - All ranges are in increasing characeter order
-/// - There are excluded characters between each pair of neighboring ranges
-#[derive(Clone, Debug, Coding)]
-pub struct CharFilter(pub Vec<RangeInclusive<char>>);
-
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
-#[extendable]
-pub enum ParserReq {
-  LexExpr(LexExpr),
-  ParseLine(ParseLine),
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ParserReq, HostExtReq)]
-pub struct LexExpr {
-  pub sys: SysId,
-  pub id: ParsId,
-  pub text: TStr,
-  pub pos: u32,
-}
-impl Request for LexExpr {
-  type Response = Option<OrcResult<LexedExpr>>;
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct LexedExpr {
-  pub pos: u32,
-  pub expr: TokenTree,
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostReq)]
-pub struct SubLex {
-  pub id: ParsId,
-  pub pos: u32,
-}
-impl Request for SubLex {
-  type Response = Option<SubLexed>;
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct SubLexed {
-  pub pos: u32,
-  pub ticket: TreeTicket,
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ParserReq, HostExtReq)]
 pub struct ParseLine {
   pub sys: SysId,
+  pub comments: Vec<Comment>,
+  pub exported: bool,
   pub line: Vec<TokenTree>,
 }
 impl Request for ParseLine {

orchid-api/src/proto.rs

@@ -27,17 +27,16 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::logging::{self, LogStrategy};
+use crate::{atom, expr, interner, lexer, logging, macros, parser, system, tree, vfs};
-use crate::{atom, expr, interner, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader {
-  pub log_strategy: LogStrategy,
+  pub log_strategy: logging::LogStrategy,
 }
 impl Decode for HostHeader {
   fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, HOST_INTRO);
-    Self { log_strategy: LogStrategy::decode(read) }
+    Self { log_strategy: logging::LogStrategy::decode(read) }
   }
 }
 impl Encode for HostHeader {
@@ -79,8 +78,10 @@ pub enum ExtHostReq {
   Ping(Ping),
   IntReq(interner::IntReq),
   Fwd(atom::Fwd),
+  SysFwd(system::SysFwd),
   ExprReq(expr::ExprReq),
-  SubLex(parser::SubLex),
+  SubLex(lexer::SubLex),
+  RunMacros(macros::RunMacros),
 }
 
 /// Notifications sent from the extension to the host
@@ -107,9 +108,11 @@ pub enum HostExtReq {
   Sweep(interner::Sweep),
   AtomReq(atom::AtomReq),
   DeserAtom(atom::DeserAtom),
-  ParserReq(parser::ParserReq),
+  LexExpr(lexer::LexExpr),
+  ParseLine(parser::ParseLine),
   GetMember(tree::GetMember),
   VfsReq(vfs::VfsReq),
+  ApplyMacro(macros::ApplyMacro),
 }
 
 /// Notifications sent from the host to the extension
@@ -147,13 +150,12 @@ impl MsgSet for HostMsgSet {
 mod tests {
   use orchid_api_traits::enc_vec;
   use ordered_float::NotNan;
-  use system::{SysDeclId, SystemDecl};
 
   use super::*;
 
   #[test]
   fn host_header_enc() {
-    let hh = HostHeader { log_strategy: LogStrategy::File("SomeFile".to_string()) };
+    let hh = HostHeader { log_strategy: logging::LogStrategy::File("SomeFile".to_string()) };
     let mut enc = &enc_vec(&hh)[..];
     eprintln!("Encoded to {enc:?}");
     HostHeader::decode(&mut enc);
@@ -164,8 +166,8 @@ mod tests {
   fn ext_header_enc() {
     let eh = ExtensionHeader {
       name: "my_extension".to_string(),
-      systems: vec![SystemDecl {
+      systems: vec![system::SystemDecl {
-        id: SysDeclId(1.try_into().unwrap()),
+        id: system::SysDeclId(1.try_into().unwrap()),
         name: "misc".to_string(),
         depends: vec!["std".to_string()],
         priority: NotNan::new(1f64).unwrap(),

orchid-api/src/system.rs

@@ -5,10 +5,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::interner::TStr;
+use crate::{CharFilter, ExtHostReq, HostExtNotif, HostExtReq, MemberKind, TStr};
-use crate::parser::CharFilter;
-use crate::proto::{HostExtNotif, HostExtReq};
-use crate::tree::MemberKind;
 
 /// ID of a system type
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -72,9 +69,24 @@ pub struct SystemInst {
 #[extends(HostExtNotif)]
 pub struct SystemDrop(pub SysId);
 
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(SysReq, HostExtReq)]
+pub struct SysFwded(pub SysId, pub Vec<u8>);
+impl Request for SysFwded {
+  type Response = Vec<u8>;
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct SysFwd(pub SysId, pub Vec<u8>);
+impl Request for SysFwd {
+  type Response = Vec<u8>;
+}
+
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 #[extendable]
 pub enum SysReq {
   NewSystem(NewSystem),
+  SysFwded(SysFwded),
 }

orchid-api/src/tree.rs

@@ -4,13 +4,11 @@ use std::sync::Arc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
+use ordered_float::NotNan;
 
-use crate::error::OrcError;
+use crate::{
-use crate::interner::{TStr, TStrv};
+  Atom, Expression, HostExtReq, Location, MacroBlock, OrcError, Placeholder, SysId, TStr, TStrv,
-use crate::location::Location;
+};
-use crate::proto::HostExtReq;
-use crate::system::SysId;
-use crate::{Atom, Expr};
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
 /// the lexer can include it in its output or discard it by implication.
@@ -31,7 +29,7 @@ pub struct TokenTree {
 #[derive(Clone, Debug, Coding)]
 pub enum Token {
   /// Lambda function head, from the opening \ until the beginning of the body.
-  Lambda(Vec<TokenTree>),
+  LambdaHead(Vec<TokenTree>),
   /// A name segment or an operator.
   Name(TStr),
   /// ::
@@ -49,9 +47,13 @@ pub enum Token {
   Bottom(Vec<OrcError>),
   /// A comment
   Comment(Arc<String>),
+  /// Placeholder
+  Ph(Placeholder),
+  /// Macro block head
+  Macro(Option<NotNan<f64>>),
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub enum Paren {
   Round,
   Square,
@@ -64,45 +66,42 @@ pub struct TreeId(pub NonZeroU64);
 #[derive(Clone, Debug, Coding)]
 pub struct Item {
   pub location: Location,
-  pub comments: Vec<(Arc<String>, Location)>,
+  pub comments: Vec<Comment>,
   pub kind: ItemKind,
 }
 
 #[derive(Clone, Debug, Coding)]
 pub enum ItemKind {
   Member(Member),
-  Raw(Vec<TokenTree>),
+  Macro(MacroBlock),
   Export(TStr),
-  Import(CompName),
+  Import(TStrv),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Comment {
+  pub text: TStr,
+  pub location: Location,
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct Member {
   pub name: TStr,
-  pub exported: bool,
   pub kind: MemberKind,
 }
 
 #[derive(Clone, Debug, Coding)]
 pub enum MemberKind {
-  Const(Expr),
+  Const(Expression),
   Module(Module),
   Lazy(TreeId),
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct Module {
-  pub imports: Vec<TStrv>,
   pub items: Vec<Item>,
 }
 
-#[derive(Clone, Debug, Coding)]
-pub struct CompName {
-  pub path: Vec<TStr>,
-  pub name: Option<TStr>,
-  pub location: Location,
-}
-
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct GetMember(pub SysId, pub TreeId);

orchid-base/Cargo.toml

@@ -12,6 +12,7 @@ hashbrown = "0.14.3"
 itertools = "0.13.0"
 lazy_static = "1.4.0"
 never = "0.1.0"
+num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }

orchid-base/src/api_utils.rs

@@ -1 +0,0 @@
-

orchid-base/src/error.rs

@@ -1,3 +1,5 @@
+use std::fmt;
+use std::ops::Add;
 use std::sync::Arc;
 
 use itertools::Itertools;
@@ -65,16 +67,86 @@ impl PartialEq for OrcErr {
 impl From<OrcErr> for Vec<OrcErr> {
   fn from(value: OrcErr) -> Self { vec![value] }
 }
-
+impl fmt::Display for OrcErr {
-pub fn errv_to_apiv<'a>(errv: impl IntoIterator<Item = &'a OrcErr>) -> Vec<api::OrcError> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-  errv.into_iter().map(OrcErr::to_api).collect_vec()
+    let pstr = self.positions.iter().map(|p| format!("{p:?}")).join("; ");
+    write!(f, "{}: {} @ {}", self.description, self.message, pstr)
+  }
 }
 
-pub fn errv_from_apiv<'a>(err: impl IntoIterator<Item = &'a api::OrcError>) -> Vec<OrcErr> {
+#[derive(Clone, Debug)]
-  err.into_iter().map(OrcErr::from_api).collect()
+pub struct EmptyErrv;
+impl fmt::Display for EmptyErrv {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "OrcErrv must not be empty")
+  }
 }
 
-pub type OrcRes<T> = Result<T, Vec<OrcErr>>;
+#[derive(Clone, Debug)]
+pub struct OrcErrv(Vec<OrcErr>);
+impl OrcErrv {
+  pub fn new(errors: impl IntoIterator<Item = OrcErr>) -> Result<Self, EmptyErrv> {
+    let v = errors.into_iter().collect_vec();
+    if v.is_empty() { Err(EmptyErrv) } else { Ok(Self(v)) }
+  }
+  #[must_use]
+  pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect_vec() }
+  #[must_use]
+  pub fn from_api<'a>(apiv: impl IntoIterator<Item = &'a api::OrcError>) -> Self {
+    let v = apiv.into_iter().map(OrcErr::from_api).collect_vec();
+    assert!(!v.is_empty(), "Error condition with 0 errors");
+    Self(v)
+  }
+  #[must_use]
+  pub fn extended<T>(mut self, errors: impl IntoIterator<Item = T>) -> Self
+  where Self: Extend<T> {
+    self.extend(errors);
+    self
+  }
+  #[must_use]
+  pub fn len(&self) -> usize { self.0.len() }
+  #[must_use]
+  pub fn is_empty(&self) -> bool { self.len() == 0 }
+  #[must_use]
+  pub fn any(&self, f: impl FnMut(&OrcErr) -> bool) -> bool { self.0.iter().any(f) }
+  #[must_use]
+  pub fn keep_only(self, f: impl FnMut(&OrcErr) -> bool) -> Option<Self> {
+    let v = self.0.into_iter().filter(f).collect_vec();
+    if v.is_empty() { None } else { Some(Self(v)) }
+  }
+  #[must_use]
+  pub fn one(&self) -> Option<&OrcErr> { (self.0.len() == 1).then(|| &self.0[9]) }
+  pub fn pos_iter(&self) -> impl Iterator<Item = ErrPos> + '_ {
+    self.0.iter().flat_map(|e| e.positions.iter().cloned())
+  }
+}
+impl From<OrcErr> for OrcErrv {
+  fn from(value: OrcErr) -> Self { Self(vec![value]) }
+}
+impl Add for OrcErrv {
+  type Output = Self;
+  fn add(self, rhs: Self) -> Self::Output { Self(self.0.into_iter().chain(rhs.0).collect_vec()) }
+}
+impl Extend<OrcErr> for OrcErrv {
+  fn extend<T: IntoIterator<Item = OrcErr>>(&mut self, iter: T) { self.0.extend(iter) }
+}
+impl Extend<OrcErrv> for OrcErrv {
+  fn extend<T: IntoIterator<Item = OrcErrv>>(&mut self, iter: T) {
+    self.0.extend(iter.into_iter().flatten())
+  }
+}
+impl IntoIterator for OrcErrv {
+  type IntoIter = std::vec::IntoIter<OrcErr>;
+  type Item = OrcErr;
+  fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
+}
+impl fmt::Display for OrcErrv {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.0.iter().join("\n"))
+  }
+}
+
+pub type OrcRes<T> = Result<T, OrcErrv>;
 
 pub fn mk_err(
   description: Tok<String>,
@@ -88,6 +160,14 @@ pub fn mk_err(
   }
 }
 
-pub trait Reporter {
+pub fn mk_errv(
-  fn report(&self, e: OrcErr);
+  description: Tok<String>,
+  message: impl AsRef<str>,
+  posv: impl IntoIterator<Item = ErrPos>,
+) -> OrcErrv {
+  mk_err(description, message, posv).into()
+}
+
+pub trait Reporter {
+  fn report(&self, e: impl Into<OrcErrv>);
 }

orchid-base/src/interner.rs

@@ -3,13 +3,14 @@ use std::hash::BuildHasher as _;
 use std::num::NonZeroU64;
 use std::ops::{Deref, DerefMut};
 use std::sync::{atomic, Arc, Mutex, MutexGuard};
-use std::{fmt, hash};
+use std::{fmt, hash, mem};
 
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools as _;
 use orchid_api_traits::{Decode, Encode, Request};
 
 use crate::api;
+use orchid_api_traits::{ApiEquiv, FromApi, ToApi};
 use crate::reqnot::{DynRequester, Requester};
 
 /// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
@@ -62,7 +63,7 @@ impl<T: Interned + Decode> Decode for Tok<T> {
   fn decode<R: std::io::Read + ?Sized>(read: &mut R) -> Self { intern(&T::decode(read)) }
 }
 
-pub trait Interned: Eq + hash::Hash + Clone + Internable<Interned = Self> {
+pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug + Internable<Interned = Self> {
   type Marker: InternMarker<Interned = Self> + Sized;
   fn intern(
     self: Arc<Self>,
@@ -71,7 +72,7 @@ pub trait Interned: Eq + hash::Hash + Clone + Internable<Interned = Self> {
   fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
 }
 
-pub trait Internable {
+pub trait Internable: fmt::Debug {
   type Interned: Interned;
   fn get_owned(&self) -> Arc<Self::Interned>;
 }
@@ -96,7 +97,6 @@ impl Interned for String {
   }
   fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.strings }
 }
-
 impl InternMarker for api::TStr {
   type Interned = String;
   fn resolve(
@@ -108,17 +108,27 @@ impl InternMarker for api::TStr {
   fn get_id(self) -> NonZeroU64 { self.0 }
   fn from_id(id: NonZeroU64) -> Self { Self(id) }
 }
-
 impl Internable for str {
   type Interned = String;
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_string()) }
 }
-
 impl Internable for String {
   type Interned = String;
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_string()) }
 }
 
+impl ApiEquiv for Tok<String> {
+  type Api = api::TStr;
+}
+impl ToApi for Tok<String> {
+  type Ctx = ();
+  fn to_api(&self, _: &mut Self::Ctx) -> Self::Api { self.marker() }
+}
+impl FromApi for Tok<String> {
+  type Ctx = ();
+  fn from_api(api: &Self::Api, _: &mut Self::Ctx) -> Self { deintern(*api) }
+}
+
 impl Interned for Vec<Tok<String>> {
   type Marker = api::TStrv;
   fn intern(
@@ -129,7 +139,6 @@ impl Interned for Vec<Tok<String>> {
   }
   fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
 }
-
 impl InternMarker for api::TStrv {
   type Interned = Vec<Tok<String>>;
   fn resolve(
@@ -143,30 +152,37 @@ impl InternMarker for api::TStrv {
   fn get_id(self) -> NonZeroU64 { self.0 }
   fn from_id(id: NonZeroU64) -> Self { Self(id) }
 }
-
 impl Internable for [Tok<String>] {
   type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_vec()) }
 }
-
 impl Internable for Vec<Tok<String>> {
   type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_vec()) }
 }
-
 impl Internable for Vec<api::TStr> {
   type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> {
     Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
   }
 }
-
 impl Internable for [api::TStr] {
   type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> {
     Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
   }
 }
+impl ApiEquiv for Tok<Vec<Tok<String>>> {
+  type Api = api::TStrv;
+}
+impl ToApi for Tok<Vec<Tok<String>>> {
+  type Ctx = ();
+  fn to_api(&self, _: &mut Self::Ctx) -> Self::Api { self.marker() }
+}
+impl FromApi for Tok<Vec<Tok<String>>> {
+  type Ctx = ();
+  fn from_api(api: &Self::Api, _: &mut Self::Ctx) -> Self { deintern(*api) }
+}
 
 /// The number of references held to any token by the interner.
 const BASE_RC: usize = 3;
@@ -262,8 +278,10 @@ pub fn init_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) {
 }
 
 pub fn intern<T: Interned>(t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
-  let mut g = interner();
   let data = t.get_owned();
+  let mut g = interner();
+  let job = format!("{t:?} in {}", if g.master.is_some() { "replica" } else { "master" });
+  eprintln!("Interning {job}");
   let typed = T::bimap(&mut g.interners);
   if let Some(tok) = typed.by_value(&data) {
     return tok;
@@ -275,6 +293,8 @@ pub fn intern<T: Interned>(t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<
   };
   let tok = Tok::new(data, marker);
   T::bimap(&mut g.interners).insert(tok.clone());
+  mem::drop(g);
+  eprintln!("Interned {job}");
   tok
 }
 

orchid-base/src/lib.rs

@@ -5,8 +5,6 @@ pub mod clone;
 pub mod combine;
 pub mod event;
 pub mod msg;
-// pub mod gen;
-pub mod api_utils;
 pub mod box_cow;
 pub mod char_filter;
 pub mod error;
@@ -18,7 +16,10 @@ pub mod logging;
 pub mod name;
 pub mod number;
 pub mod parse;
+pub mod pure_seq;
 pub mod reqnot;
 pub mod sequence;
 pub mod tokens;
 pub mod tree;
+pub mod macros;
+mod match_mapping;

orchid-base/src/location.rs

@@ -1,15 +1,16 @@
 //! Structures that show where code or semantic elements came from
 
+use crate::match_mapping;
 use std::fmt;
 use std::hash::Hash;
 use std::ops::Range;
-use std::sync::Arc;
 
 use trait_set::trait_set;
 
-use crate::interner::{deintern, Tok};
+use orchid_api_traits::{ApiEquiv, FromApi, ToApi};
+use crate::interner::{deintern, intern, Tok};
 use crate::name::Sym;
-use crate::{api, sym};
+use crate::{api, intern, sym};
 
 trait_set! {
   pub trait GetSrc = FnMut(&Sym) -> Tok<String>;
@@ -18,6 +19,7 @@ trait_set! {
 #[derive(Debug, Clone)]
 pub enum Pos {
   None,
+  SlotTarget,
   /// Used in functions to denote the generated code that carries on the
   /// location of the call. Not allowed in the const tree.
   Inherit,
@@ -28,24 +30,6 @@ pub enum Pos {
   Range(Range<u32>),
 }
 impl Pos {
-  pub fn to_api(&self) -> api::Location {
-    match self {
-      Self::Inherit => api::Location::Inherit,
-      Self::None => api::Location::None,
-      Self::Range(r) => api::Location::Range(r.clone()),
-      Self::Gen(cgi) => api::Location::Gen(cgi.to_api()),
-      Self::SourceRange(sr) => api::Location::SourceRange(sr.to_api()),
-    }
-  }
-  pub fn from_api(loc: &api::Location) -> Self {
-    match loc {
-      api::Location::Inherit => Self::Inherit,
-      api::Location::None => Self::None,
-      api::Location::Range(r) => Self::Range(r.clone()),
-      api::Location::Gen(cgi) => CodeGenInfo::from_api(cgi).location(),
-      api::Location::SourceRange(sr) => SourceRange::from_api(sr).location(),
-    }
-  }
   pub fn pretty_print(&self, get_src: &mut impl GetSrc) -> String {
     match self {
       Self::Gen(g) => g.to_string(),
@@ -55,6 +39,32 @@ impl Pos {
     }
   }
 }
+impl ApiEquiv for Pos {
+  type Api = api::Location;
+}
+impl FromApi for Pos {
+  type Ctx = ();
+  fn from_api(api: &Self::Api, ctx: &mut Self::Ctx) -> Self {
+    match_mapping!(api, api::Location => Pos {
+      None, Inherit, SlotTarget,
+      Range(r.clone()),
+      Gen(cgi => CodeGenInfo::from_api(cgi, &mut ())),
+      SourceRange(sr => CodeGenInfo::from_api(sr, &mut ()))
+    })
+  }
+}
+
+impl ToApi for Pos {
+  type Ctx = ();
+  fn to_api(&self, ctx: &mut Self::Ctx) -> Self::Api {
+    match_mapping!(self, Pos => Self::Api {
+      None, Inherit, SlotTarget,
+      Range(r.clone()),
+      Gen(cgi.to_api(ctx)),
+      SourceRange(sr.to_api(ctx)),
+    })
+  }
+}
 
 /// Exact source code location. Includes where the code was loaded from, what
 /// the original source code was, and a byte range.
@@ -67,12 +77,6 @@ impl SourceRange {
   pub fn new(range: &Range<u32>, path: &Sym) -> Self {
     Self { range: range.clone(), path: path.clone() }
   }
-  pub fn to_api(&self) -> api::SourceRange {
-    api::SourceRange { path: self.path.tok().marker(), range: self.range.clone() }
-  }
-  pub fn from_api(sr: &api::SourceRange) -> Self {
-    Self { path: Sym::from_tok(deintern(sr.path)).unwrap(), range: sr.range.clone() }
-  }
   /// Create a dud [SourceRange] for testing. Its value is unspecified and
   /// volatile.
   pub fn mock() -> Self { Self { range: 0..1, path: sym!(test) } }
@@ -85,7 +89,7 @@ impl SourceRange {
   /// 0-based index of last byte + 1
   pub fn end(&self) -> u32 { self.range.end }
   /// Syntactic location
-  pub fn location(&self) -> Pos { Pos::SourceRange(self.clone()) }
+  pub fn pos(&self) -> Pos { Pos::SourceRange(self.clone()) }
   /// Transform the numeric byte range
   pub fn map_range(&self, map: impl FnOnce(Range<u32>) -> Range<u32>) -> Self {
     Self { range: map(self.range()), path: self.path() }
@@ -99,6 +103,24 @@ impl SourceRange {
       (false, _) => format!("{sl}:{sc}..{el}:{ec}"),
     }
   }
+  pub fn zw(path: Sym, pos: u32) -> Self {
+    Self { path, range: pos..pos }
+  }
+}
+impl ApiEquiv for SourceRange {
+  type Api = api::SourceRange;
+}
+impl FromApi for SourceRange {
+  type Ctx = ();
+  fn from_api(api: &Self::Api, ctx: &mut Self::Ctx) -> Self {
+    Self { path: Sym::from_api(&api.path, ctx), range: api.range.clone() }
+  }
+}
+impl ToApi for SourceRange {
+  type Ctx = ();
+  fn to_api(&self, ctx: &mut Self::Ctx) -> Self::Api {
+    api::SourceRange { path: self.path.to_api(ctx), range: self.range.clone() }
+  }
 }
 
 /// Information about a code generator attached to the generated code
@@ -107,26 +129,17 @@ pub struct CodeGenInfo {
   /// formatted like a Rust namespace
   pub generator: Sym,
   /// Unformatted user message with relevant circumstances and parameters
-  pub details: Arc<String>,
+  pub details: Tok<String>,
 }
 impl CodeGenInfo {
   /// A codegen marker with no user message and parameters
-  pub fn no_details(generator: Sym) -> Self { Self { generator, details: Arc::new(String::new()) } }
+  pub fn no_details(generator: Sym) -> Self { Self { generator, details: intern!(str: "") } }
   /// A codegen marker with a user message or parameters
   pub fn details(generator: Sym, details: impl AsRef<str>) -> Self {
-    Self { generator, details: Arc::new(details.as_ref().to_string()) }
+    Self { generator, details: intern(details.as_ref()) }
   }
   /// Syntactic location
-  pub fn location(&self) -> Pos { Pos::Gen(self.clone()) }
+  pub fn pos(&self) -> Pos { Pos::Gen(self.clone()) }
-  pub fn to_api(&self) -> api::CodeGenInfo {
-    api::CodeGenInfo { generator: self.generator.tok().marker(), details: self.details.to_string() }
-  }
-  pub fn from_api(cgi: &api::CodeGenInfo) -> Self {
-    Self {
-      generator: Sym::from_tok(deintern(cgi.generator)).unwrap(),
-      details: Arc::new(cgi.details.clone()),
-    }
-  }
 }
 impl fmt::Debug for CodeGenInfo {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeGenInfo({self})") }
@@ -137,6 +150,24 @@ impl fmt::Display for CodeGenInfo {
     if !self.details.is_empty() { write!(f, ", details: {}", self.details) } else { write!(f, ".") }
   }
 }
+impl ApiEquiv for CodeGenInfo {
+  type Api = api::CodeGenInfo;
+}
+impl FromApi for CodeGenInfo {
+  type Ctx = ();
+  fn from_api(api: &Self::Api, ctx: &mut Self::Ctx) -> Self {
+    Self {
+      generator: Sym::from_api(&api.generator, ctx),
+      details: Tok::from_api(&api.details, ctx),
+    }
+  }
+}
+impl ToApi for CodeGenInfo {
+  type Ctx = ();
+  fn to_api(&self, ctx: &mut Self::Ctx) -> Self::Api {
+    api::CodeGenInfo { generator: self.generator.to_api(ctx), details: self.details.to_api(ctx) }
+  }
+}
 
 #[must_use]
 fn pos2lc(s: &str, i: u32) -> (u32, u32) {

orchid-base/src/logging.rs

@@ -14,7 +14,7 @@ impl Logger {
   pub fn log(&self, msg: impl AsRef<str>) { writeln!(self, "{}", msg.as_ref()) }
   pub fn strat(&self) -> api::LogStrategy { self.0.clone() }
   pub fn log_buf(&self, event: impl AsRef<str>, buf: &[u8]) {
-    if !std::env::var("ORCHID_LOG_BUFFERS").unwrap().is_empty() {
+    if std::env::var("ORCHID_LOG_BUFFERS").is_ok_and(|v| !v.is_empty()) {
       writeln!(self, "{}: [{}]", event.as_ref(), buf.iter().map(|b| format!("{b:02x}")).join(" "))
     }
   }

orchid-base/src/macros.rs

@@ -0,0 +1,61 @@
+use itertools::Itertools;
+
+use crate::{name::Sym, tree::{AtomTok, Paren, Ph, TokTree}};
+use std::marker::PhantomData;
+
+use crate::{api, location::Pos};
+
+#[derive(Clone, Debug)]
+pub struct MacroSlot<'a>(api::MacroTreeId, PhantomData<&'a ()>);
+
+#[derive(Clone, Debug)]
+pub struct MTree<'a, A: AtomTok> {
+  pub pos: Pos,
+  pub tok: MTok<'a, A>
+}
+
+#[derive(Clone, Debug)]
+pub enum MTok<'a, A> {
+  S(Paren, Vec<MTree<'a, A>>),
+  Name(Sym),
+  Slot(MacroSlot<'a>),
+  Lambda(Vec<MTree<'a, A>>, Vec<MTree<'a, A>>),
+  Ph(Ph),
+  Atom(A)
+}
+impl<'a, A> MTree<'a, A> {
+  pub(crate) fn from_api(api: &api::MacroTree) -> Self {
+    use api::MacroToken as MTK;
+    let tok = match &api.token {
+      MTK::Lambda(x, b) => MTok::Lambda(mtreev_from_api(x), mtreev_from_api(b)),
+      MTK::Name(t) => MTok::Name(Sym::deintern(*t)),
+      MTK::Slot(tk) => MTok::Slot(MacroSlot(tk.clone(), PhantomData)),
+      MTK::S(p, b) => MTok::S(p.clone(), mtreev_from_api(b)),
+      MTK::Ph(ph) => MTok::Ph(Ph::from_api(ph)),
+    };
+    Self { pos: Pos::from_api(&api.location), tok }
+  }
+  pub(crate) fn to_api(&self) -> api::MacroTree {
+    use api::MacroToken as MTK;
+    let token = match &self.tok {
+      MTok::Lambda(x, b) => MTK::Lambda(mtreev_to_api(x), mtreev_to_api(b)),
+      MTok::Name(t) => MTK::Name(t.tok().marker()),
+      MTok::Ph(ph) => MTK::Ph(ph.to_api()),
+      MTok::S(p, b) => MTK::S(p.clone(), mtreev_to_api(b)),
+      MTok::Slot(tk) => MTK::Slot(tk.0.clone()),
+    };
+    api::MacroTree { location: self.pos.to_api(), token }
+  }
+}
+
+pub fn mtreev_from_api<'a, 'b, A>(
+  api: impl IntoIterator<Item = &'b api::MacroTree>
+) -> Vec<MTree<'a, A>> {
+  api.into_iter().map(MTree::from_api).collect_vec()
+}
+
+pub fn mtreev_to_api<'a: 'b, 'b, A: 'b>(
+  v: impl IntoIterator<Item = &'b MTree<'a, A>>
+) -> Vec<api::MacroTree> {
+  v.into_iter().map(MTree::to_api).collect_vec()
+}

orchid-base/src/match_mapping.rs

@@ -0,0 +1,95 @@
+/// A shorthand for mapping over enums with identical structure. Used for converting between
+/// owned enums and the corresponding API enums that only differ in the type of their
+/// fields.
+/// 
+/// The basic form is
+/// ```ignore
+/// match_mapping!(self, ThisType => OtherType {
+///   EmptyVariant,
+///   TupleVariant(foo => intern(foo), bar.clone()),
+///   StructVariant{ a.to_api(), b => b.}
+/// })
+/// ```
+#[macro_export]
+macro_rules! match_mapping {
+  ($input:expr, $src:ty => $tgt:ty {
+    $($branches:tt)*
+  }) => {
+    match_mapping!(@BRANCH_MUNCH (($input) ($src) ($tgt)) () $($branches)* ,)
+  };
+  (@BRANCHES_DONE ( ($input:expr) ($src:ty) ($tgt:ty) )
+    $( ( $variant:ident $($pat:tt)*) )*
+  ) => {
+    {
+      use $src as Foo;
+      match $input {
+        $(
+          match_mapping!(@PAT (Foo :: $variant) $($pat)*) =>
+            match_mapping!(@VAL (< $tgt >:: $variant) $($pat)*),
+        )*
+      }
+    }
+  };
+  (@BRANCH_MUNCH $ext:tt ( $($branches:tt)* ) $(,)?) => {
+    match_mapping!(@BRANCHES_DONE $ext $($branches)* )
+  };
+  (@BRANCH_MUNCH $ext:tt ( $($branches:tt)* ) $variant:ident , $($tail:tt)*) => {
+    match_mapping!(@BRANCH_MUNCH $ext ( $($branches)* ($variant) ) $($tail)*)
+  };
+  (@BRANCH_MUNCH $ext:tt ( $($branches:tt)* ) $variant:ident $pat:tt , $($tail:tt)*) => {
+    match_mapping!(@BRANCH_MUNCH $ext
+      ( $($branches)* ($variant $pat) )
+    $($tail)*)
+  };
+  (@PAT ($($prefix:tt)*) ( $($fields:tt)* )) => {
+    $($prefix)* ( match_mapping!(@PAT_MUNCH () $($fields)*) )
+  };
+  (@PAT ($($prefix:tt)*) { $($fields:tt)* }) => {
+    $($prefix)* { match_mapping!(@PAT_MUNCH () $($fields)*) }
+  };
+  (@PAT ($($path:tt)*)) => { $($path)* };
+  (@PAT_MUNCH ($($names:ident)*) $name:ident => $value:expr) => { $($names ,)* $name };
+  (@PAT_MUNCH ($($names:ident)*) $name:ident => $value:expr , $($tail:tt)*) => {
+    match_mapping!(@PAT_MUNCH ($($names)* $name) $($tail)*)
+  };
+  (@PAT_MUNCH ($($names:ident)*) $name:ident . $($tail:tt)*) => { 
+    match_mapping!(@PAT_DOT_MUNCH ($($names)* $name) $($tail)*)
+  };
+  (@PAT_MUNCH ($($names:ident)*)) => { $($names),* };
+  (@PAT_DOT_MUNCH $names:tt , $($tail:tt)*) => {
+    match_mapping!(@PAT_MUNCH $names $($tail)*)
+  };
+  (@PAT_DOT_MUNCH $names:tt $_:tt $($tail:tt)*) => {
+    match_mapping!(@PAT_DOT_MUNCH $names $($tail)*)
+  };
+  (@PAT_DOT_MUNCH ($($names:tt)*)) => { $($names),* };
+  (@VAL ($($prefix:tt)*)) => { $($prefix)* };
+  (@VAL ($($prefix:tt)*) ( $($fields:tt)* )) => {
+    $($prefix)* ( match_mapping!(@VAL_MUNCH () () $($fields)* ) )
+  };
+  (@VAL ($($prefix:tt)*) { $($fields:tt)* }) => {
+    $($prefix)* { match_mapping!(@VAL_MUNCH {} () $($fields)* ) }
+  };
+  (@VAL_MUNCH () ($($prefix:tt)*) $name:ident => $value:expr) => { $($prefix)* $value };
+  (@VAL_MUNCH () ($($prefix:tt)*) $name:ident => $value:expr , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH () ($($prefix)* $value, ) $($tail)*)
+  };
+  (@VAL_MUNCH {} ($($prefix:tt)*) $name:ident => $value:expr) => { $($prefix)* $name: $value };
+  (@VAL_MUNCH {} ($($prefix:tt)*) $name:ident => $value:expr , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH {} ($($prefix)* $name: $value, ) $($tail)*)
+  };
+  (@VAL_MUNCH () ($($prefix:tt)*) $name:ident . $member:tt $($tail:tt)*) => {
+    match_mapping!(@VAL_DOT_MUNCH () ($($prefix)* $name . $member ) $($tail)*)
+  };
+  (@VAL_MUNCH {} ($($prefix:tt)*) $name:ident . $member:tt $($tail:tt)*) => {
+    match_mapping!(@VAL_DOT_MUNCH {} ($($prefix)* $name: $name . $member) $($tail)*)
+  };
+  (@VAL_DOT_MUNCH $ptyp:tt ($($prefix:tt)*) , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH $ptyp ($($prefix)* ,) $($tail)*)
+  };
+  (@VAL_DOT_MUNCH $ptyp:tt ($($prefix:tt)*) $tt:tt $($tail:tt)*) => {
+    match_mapping!(@VAL_DOT_MUNCH $ptyp ($($prefix)* $tt) $($tail)*)
+  };
+  (@VAL_DOT_MUNCH $ptyp:tt ($($prefix:tt)*)) => { $($prefix)* };
+  (@VAL_MUNCH $_ptyp:tt ($($prefix:tt)*)) => { $($prefix)* };
+}

orchid-base/src/name.rs

@@ -9,10 +9,10 @@ use std::path::Path;
 use std::{fmt, slice, vec};
 
 use itertools::Itertools;
-use orchid_api::TStrv;
 use trait_set::trait_set;
 
 use crate::api;
+use crate::api_conv::{ApiEquiv, FromApi, ToApi};
 use crate::interner::{deintern, intern, InternMarker, Tok};
 
 trait_set! {
@@ -357,7 +357,7 @@ impl Sym {
   pub fn id(&self) -> NonZeroU64 { self.0.marker().get_id() }
   /// Extern the sym for editing
   pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
-  pub fn deintern(marker: TStrv) -> Sym {
+  pub fn deintern(marker: api::TStrv) -> Sym {
     Self::from_tok(deintern(marker)).expect("Empty sequence found for serialized Sym")
   }
 }
@@ -386,6 +386,17 @@ impl Deref for Sym {
   type Target = PathSlice;
   fn deref(&self) -> &Self::Target { self.borrow() }
 }
+impl ApiEquiv for Sym {
+  type Api = api::TStrv;
+}
+impl<C> ToApi<C> for Sym {
+  fn to_api(&self, ctx: &mut C) -> Self::Api { self.tok().to_api(ctx) }
+}
+impl<C> FromApi<C> for Sym {
+  fn from_api(api: &Self::Api, ctx: &mut C) -> Self {
+    Self::from_tok(Tok::from_api(api, ctx)).expect("Empty sequence found for serialized Sym")
+  }
+}
 
 /// An abstraction over tokenized vs non-tokenized names so that they can be
 /// handled together in datastructures. The names can never be empty

orchid-base/src/number.rs

@@ -3,6 +3,7 @@ use std::ops::Range;
 
 use ordered_float::NotNan;
 use rust_decimal::Decimal;
+use num_traits::ToPrimitive;
 
 use crate::error::{mk_err, OrcErr};
 use crate::intern;
@@ -21,6 +22,16 @@ pub enum Numeric {
 impl Numeric {
   pub fn decimal(num: i64, scale: u32) -> Self { Self::Decimal(Decimal::new(num, scale)) }
   pub fn float(value: f64) -> Self { Self::Float(NotNan::new(value).unwrap()) }
+  pub fn to_f64(self) -> NotNan<f64> {
+    match self {
+      Self::Float(f) => f,
+      Self::Decimal(d) => {
+        let f = d.to_f64().expect("This is apparently always possible");
+        NotNan::new(f).expect("decimal was nan")
+      },
+      Self::Uint(i) => NotNan::new(i as f64).expect("int cannot be NaN"),
+    }
+  }
 }
 
 /// Rasons why [parse_num] might fail. See [NumError].

orchid-base/src/parse.rs

@@ -1,14 +1,13 @@
+use std::iter;
 use std::ops::{Deref, Range};
-use std::sync::Arc;
-use std::{fmt, iter};
 
 use itertools::Itertools;
 
-use crate::error::{mk_err, OrcRes, Reporter};
+use crate::error::{mk_err, mk_errv, OrcRes, Reporter};
-use crate::interner::{deintern, Tok};
+use crate::interner::{deintern, intern, Tok};
 use crate::location::Pos;
 use crate::name::VPath;
-use crate::tree::{AtomInTok, Paren, TokTree, Token};
+use crate::tree::{AtomTok, ExtraTok, Paren, TokTree, Token};
 use crate::{api, intern};
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
@@ -17,11 +16,11 @@ pub fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{
 pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 
 #[derive(Debug)]
-pub struct Snippet<'a, 'b, A: AtomInTok, X> {
+pub struct Snippet<'a, 'b, A: AtomTok, X: ExtraTok> {
   prev: &'a TokTree<'b, A, X>,
   cur: &'a [TokTree<'b, A, X>],
 }
-impl<'a, 'b, A: AtomInTok, X> Snippet<'a, 'b, A, X> {
+impl<'a, 'b, A: AtomTok, X: ExtraTok> Snippet<'a, 'b, A, X> {
   pub fn new(prev: &'a TokTree<'b, A, X>, cur: &'a [TokTree<'b, A, X>]) -> Self {
     Self { prev, cur }
   }
@@ -44,6 +43,9 @@ impl<'a, 'b, A: AtomInTok, X> Snippet<'a, 'b, A, X> {
   pub fn pop_front(self) -> Option<(&'a TokTree<'b, A, X>, Self)> {
     self.cur.first().map(|r| (r, self.split_at(1).1))
   }
+  pub fn pop_back(self) -> Option<(Self, &'a TokTree<'b, A, X>)> {
+    self.cur.last().map(|r| (self.split_at(self.len() - 1).0, r))
+  }
   pub fn split_once(self, f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<(Self, Self)> {
     let idx = self.find_idx(f)?;
     Some((self.split_at(idx).0, self.split_at(idx + 1).1))
@@ -65,25 +67,25 @@ impl<'a, 'b, A: AtomInTok, X> Snippet<'a, 'b, A, X> {
     self.split_at(non_fluff_start.unwrap_or(self.len())).1
   }
 }
-impl<'a, 'b, A: AtomInTok, X> Copy for Snippet<'a, 'b, A, X> {}
+impl<'a, 'b, A: AtomTok, X: ExtraTok> Copy for Snippet<'a, 'b, A, X> {}
-impl<'a, 'b, A: AtomInTok, X> Clone for Snippet<'a, 'b, A, X> {
+impl<'a, 'b, A: AtomTok, X: ExtraTok> Clone for Snippet<'a, 'b, A, X> {
   fn clone(&self) -> Self { *self }
 }
-impl<'a, 'b, A: AtomInTok, X> Deref for Snippet<'a, 'b, A, X> {
+impl<'a, 'b, A: AtomTok, X: ExtraTok> Deref for Snippet<'a, 'b, A, X> {
   type Target = [TokTree<'b, A, X>];
   fn deref(&self) -> &Self::Target { self.cur }
 }
 
 /// Remove tokens that aren't meaningful in expression context, such as comments
 /// or line breaks
-pub fn strip_fluff<'a, A: AtomInTok, X: Clone>(
+pub fn strip_fluff<'a, A: AtomTok, X: ExtraTok>(
   tt: &TokTree<'a, A, X>,
 ) -> Option<TokTree<'a, A, X>> {
   let tok = match &tt.tok {
     Token::BR => return None,
     Token::Comment(_) => return None,
     Token::LambdaHead(arg) => Token::LambdaHead(arg.iter().filter_map(strip_fluff).collect()),
-    Token::S(p, b) => Token::S(p.clone(), b.iter().filter_map(strip_fluff).collect()),
+    Token::S(p, b) => Token::S(*p, b.iter().filter_map(strip_fluff).collect()),
     t => t.clone(),
   };
   Some(TokTree { tok, range: tt.range.clone() })
@@ -91,13 +93,21 @@ pub fn strip_fluff<'a, A: AtomInTok, X: Clone>(
 
 #[derive(Clone, Debug)]
 pub struct Comment {
-  pub text: Arc<String>,
+  pub text: Tok<String>,
   pub pos: Pos,
 }
+impl Comment {
+  pub fn from_api(api: &api::Comment) -> Self {
+    Self { pos: Pos::from_api(&api.location), text: deintern(api.text) }
+  }
+  pub fn to_api(&self) -> api::Comment {
+    api::Comment { location: self.pos.to_api(), text: self.text.marker() }
+  }
+}
 
-pub fn line_items<'a, 'b, A: AtomInTok, X>(
+pub fn line_items<'a, 'b, A: AtomTok, X: ExtraTok>(
   snip: Snippet<'a, 'b, A, X>,
-) -> Vec<(Vec<Comment>, Snippet<'a, 'b, A, X>)> {
+) -> Vec<Parsed<'a, 'b, Vec<Comment>, A, X>> {
   let mut items = Vec::new();
   let mut comments = Vec::new();
   for mut line in snip.split(|t| matches!(t, Token::BR)) {
@@ -109,72 +119,79 @@ pub fn line_items<'a, 'b, A: AtomInTok, X>(
     match line.find_idx(|t| !matches!(t, Token::Comment(_))) {
       None => comments.extend(line.cur),
       Some(i) => {
-        let (cmts, line) = line.split_at(i);
+        let (cmts, tail) = line.split_at(i);
         let comments = Vec::from_iter(comments.drain(..).chain(cmts.cur).map(|t| match &t.tok {
-          Token::Comment(c) => Comment { text: c.clone(), pos: Pos::Range(t.range.clone()) },
+          Token::Comment(c) => Comment { text: intern(&**c), pos: Pos::Range(t.range.clone()) },
           _ => unreachable!("All are comments checked above"),
         }));
-        items.push((comments, line));
+        items.push(Parsed { output: comments, tail });
       },
     }
   }
   items
 }
 
-pub fn try_pop_no_fluff<'a, 'b, A: AtomInTok, X>(
+pub fn try_pop_no_fluff<'a, 'b, A: AtomTok, X: ExtraTok>(
   snip: Snippet<'a, 'b, A, X>,
-) -> OrcRes<(&'a TokTree<'b, A, X>, Snippet<'a, 'b, A, X>)> {
+) -> ParseRes<'a, 'b, &'a TokTree<'b, A, X>, A, X> {
-  snip.skip_fluff().pop_front().ok_or_else(|| {
+  snip.skip_fluff().pop_front().map(|(output, tail)| Parsed { output, tail }).ok_or_else(|| {
-    vec![mk_err(intern!(str: "Unexpected end"), "Pattern ends abruptly", [
+    mk_errv(
-      Pos::Range(snip.pos()).into()
+      intern!(str: "Unexpected end"),
-    ])]
+      "Pattern ends abruptly",
+      [Pos::Range(snip.pos()).into()],
+    )
   })
 }
 
-pub fn expect_end(snip: Snippet<'_, '_, impl AtomInTok, impl Sized>) -> OrcRes<()> {
+pub fn expect_end(snip: Snippet<'_, '_, impl AtomTok, impl ExtraTok>) -> OrcRes<()> {
   match snip.skip_fluff().get(0) {
-    Some(surplus) => Err(vec![mk_err(
+    Some(surplus) => Err(mk_errv(
       intern!(str: "Extra code after end of line"),
       "Code found after the end of the line",
       [Pos::Range(surplus.range.clone()).into()],
-    )]),
+    )),
     None => Ok(()),
   }
 }
 
-pub fn expect_tok<'a, 'b, A: AtomInTok, X: fmt::Display>(
+pub fn expect_tok<'a, 'b, A: AtomTok, X: ExtraTok>(
   snip: Snippet<'a, 'b, A, X>,
   tok: Tok<String>,
-) -> OrcRes<Snippet<'a, 'b, A, X>> {
+) -> ParseRes<'a, 'b, (), A, X> {
-  let (head, tail) = try_pop_no_fluff(snip)?;
+  let Parsed { output: head, tail } = try_pop_no_fluff(snip)?;
   match &head.tok {
-    Token::Name(n) if *n == tok => Ok(tail),
+    Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
-    t => Err(vec![mk_err(
+    t => Err(mk_errv(
       intern!(str: "Expected specific keyword"),
       format!("Expected {tok} but found {t}"),
       [Pos::Range(head.range.clone()).into()],
-    )]),
+    )),
   }
 }
 
-pub fn parse_multiname<'a, 'b, A: AtomInTok, X: fmt::Display>(
+pub struct Parsed<'a, 'b, T, A: AtomTok, X: ExtraTok> {
+  pub output: T,
+  pub tail: Snippet<'a, 'b, A, X>,
+}
+
+pub type ParseRes<'a, 'b, T, A, X> = OrcRes<Parsed<'a, 'b, T, A, X>>;
+
+pub fn parse_multiname<'a, 'b, A: AtomTok, X: ExtraTok>(
   ctx: &impl Reporter,
   tail: Snippet<'a, 'b, A, X>,
-) -> OrcRes<(Vec<CompName>, Snippet<'a, 'b, A, X>)> {
+) -> ParseRes<'a, 'b, Vec<(Import, Pos)>, A, X> {
   let ret = rec(ctx, tail);
   #[allow(clippy::type_complexity)] // it's an internal function
-  pub fn rec<'a, 'b, A: AtomInTok, X: fmt::Display>(
+  pub fn rec<'a, 'b, A: AtomTok, X: ExtraTok>(
     ctx: &impl Reporter,
     tail: Snippet<'a, 'b, A, X>,
-  ) -> OrcRes<(Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, Snippet<'a, 'b, A, X>)> {
+  ) -> ParseRes<'a, 'b, Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, A, X> {
     let comma = intern!(str: ",");
     let globstar = intern!(str: "*");
     let (name, tail) = tail.skip_fluff().pop_front().ok_or_else(|| {
-      vec![mk_err(
+      mk_err(intern!(str: "Expected name"), "Expected a name, a list of names, or a globstar.", [
-        intern!(str: "Expected name"),
+        Pos::Range(tail.pos()).into(),
-        "Expected a name, a list of names, or a globstar.",
+      ])
-        [Pos::Range(tail.pos()).into()],
-      )]
     })?;
     if let Some((Token::NS, tail)) = tail.skip_fluff().pop_front().map(|(tt, s)| (&tt.tok, s)) {
       let n = match &name.tok {
@@ -184,25 +201,27 @@ pub fn parse_multiname<'a, 'b, A: AtomInTok, X: fmt::Display>(
         ])),
       };
       match (rec(ctx, tail), n) {
-        (Err(ev), n) => Err(Vec::from_iter(ev.into_iter().chain(n.err()))),
+        (Err(ev), n) => Err(ev.extended(n.err())),
-        (Ok((_, tail)), Err(e)) => {
+        (Ok(Parsed { tail, .. }), Err(e)) => {
           ctx.report(e);
-          Ok((vec![], tail))
+          Ok(Parsed { output: vec![], tail })
         },
-        (Ok((n, tail)), Ok(pre)) =>
+        (Ok(Parsed { tail, output }), Ok(pre)) => Ok(Parsed {
-          Ok((n.into_iter().update(|i| i.0.push(pre.clone())).collect_vec(), tail)),
+          output: output.into_iter().update(|i| i.0.push(pre.clone())).collect_vec(),
+          tail,
+        }),
       }
     } else {
-      let names = match &name.tok {
+      let output = match &name.tok {
         Token::Name(ntok) => {
           let nopt = match ntok {
             n if *n == globstar => None,
             n if n.starts_with(op_char) =>
-              return Err(vec![mk_err(
+              return Err(mk_errv(
                 intern!(str: "Unescaped operator in multiname"),
                 "Operators in multinames should be enclosed in []",
                 [Pos::Range(name.range.clone()).into()],
-              )]),
+              )),
             n => Some(n.clone()),
           };
           vec![(vec![], nopt, Pos::Range(name.range.clone()))]
@@ -226,9 +245,9 @@ pub fn parse_multiname<'a, 'b, A: AtomInTok, X: fmt::Display>(
           let body = Snippet::new(name, b);
           for csent in body.split(|n| matches!(n, Token::Name(n) if *n == comma)) {
             match rec(ctx, csent) {
-              Err(e) => e.into_iter().for_each(|e| ctx.report(e)),
+              Err(e) => ctx.report(e),
-              Ok((v, surplus)) => match surplus.get(0) {
+              Ok(Parsed { output, tail }) => match tail.get(0) {
-                None => ok.extend(v),
+                None => ok.extend(output),
                 Some(t) => ctx.report(mk_err(
                   intern!(str: "Unexpected token in multiname group"),
                   "Unexpected token. Likely missing a :: or , or wanted [] instead of ()",
@@ -240,40 +259,39 @@ pub fn parse_multiname<'a, 'b, A: AtomInTok, X: fmt::Display>(
           ok
         },
         t =>
-          return Err(vec![mk_err(
+          return Err(mk_errv(
             intern!(str: "Unrecognized name end"),
             format!("Names cannot end with {t} tokens"),
             [Pos::Range(name.range.clone()).into()],
-          )]),
+          )),
       };
-      Ok((names, tail))
+      Ok(Parsed { output, tail })
     }
   }
-  ret.map(|(i, tail)| {
+  ret.map(|Parsed { output, tail }| {
-    let i = Vec::from_iter((i.into_iter()).map(|(p, name, pos)| CompName {
+    let output = (output.into_iter())
-      path: VPath::new(p.into_iter().rev()),
+      .map(|(p, name, pos)| (Import { path: VPath::new(p.into_iter().rev()), name }, pos))
-      name,
+      .collect_vec();
-      pos,
+    Parsed { output, tail }
-    }));
-    (i, tail)
   })
 }
 
 /// A compound name, possibly ending with a globstar
 #[derive(Debug, Clone)]
-pub struct CompName {
+pub struct Import {
   pub path: VPath,
   pub name: Option<Tok<String>>,
-  pub pos: Pos,
-}
-impl CompName {
-  pub fn from_api(i: api::CompName) -> Self {
-    Self {
-      path: VPath::new(i.path.into_iter().map(deintern)),
-      name: i.name.map(deintern),
-      pos: Pos::from_api(&i.location),
-    }
 }
+impl Import {
+  // pub fn from_api(i: api::CompName) -> Self {
+  //   Self { path: VPath::new(i.path.into_iter().map(deintern)), name: i.name.map(deintern) }
+  // }
+  // pub fn to_api(&self) -> api::CompName {
+  //   api::CompName {
+  //     path: self.path.iter().map(|t| t.marker()).collect(),
+  //     name: self.name.as_ref().map(|t| t.marker()),
+  //   }
+  // }
 }
 
 #[cfg(test)]
@@ -282,6 +300,14 @@ mod test {
 
   use super::Snippet;
 
-  fn _covary_snip_a<'a, 'b>(x: Snippet<'static, 'b, Never, ()>) -> Snippet<'a, 'b, Never, ()> { x }
+  fn _covary_snip_a<'a, 'b>(
-  fn _covary_snip_b<'a, 'b>(x: Snippet<'a, 'static, Never, ()>) -> Snippet<'a, 'b, Never, ()> { x }
+    x: Snippet<'static, 'b, Never, Never>,
+  ) -> Snippet<'a, 'b, Never, Never> {
+    x
+  }
+  fn _covary_snip_b<'a, 'b>(
+    x: Snippet<'a, 'static, Never, Never>,
+  ) -> Snippet<'a, 'b, Never, Never> {
+    x
+  }
 }

orchid-base/src/pure_seq.rs

@@ -0,0 +1,35 @@
+//! Methods to operate on Rust vectors in a declarative manner
+
+use std::iter;
+
+/// Pure version of [Vec::push]
+///
+/// Create a new vector consisting of the provided vector with the
+/// element appended. See [pushed_ref] to use it with a slice
+pub fn pushed<I: IntoIterator, C: FromIterator<I::Item>>(vec: I, t: I::Item) -> C {
+  vec.into_iter().chain(iter::once(t)).collect()
+}
+
+/// Pure version of [Vec::push]
+///
+/// Create a new vector consisting of the provided slice with the
+/// element appended. See [pushed] for the owned version
+pub fn pushed_ref<'a, T: Clone + 'a, C: FromIterator<T>>(
+  vec: impl IntoIterator<Item = &'a T>,
+  t: T,
+) -> C {
+  vec.into_iter().cloned().chain(iter::once(t)).collect()
+}
+
+/// Push an element on the adhoc stack, pass it to the callback, then pop the
+/// element out again.
+pub fn with_pushed<T, U>(
+  vec: &mut Vec<T>,
+  item: T,
+  cb: impl for<'a> FnOnce(&'a mut Vec<T>) -> U,
+) -> (T, U) {
+  vec.push(item);
+  let out = cb(vec);
+  let item = vec.pop().expect("top element stolen by callback");
+  (item, out)
+}

orchid-base/src/reqnot.rs

@@ -1,3 +1,5 @@
+use std::any::Any;
+use std::cell::RefCell;
 use std::marker::PhantomData;
 use std::ops::{BitAnd, Deref};
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -5,17 +7,24 @@ use std::sync::mpsc::{sync_channel, SyncSender};
 use std::sync::{Arc, Mutex};
 use std::{mem, thread};
 
+use derive_destructure::destructure;
 use dyn_clone::{clone_box, DynClone};
 use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
 
-pub struct ReplyToken;
+pub struct Receipt;
+impl Receipt {
+  pub fn off_thread(name: String, cb: impl FnOnce() -> Self + Send + 'static) -> Self {
+    thread::Builder::new().name(name).spawn(cb).unwrap();
+    Self
+  }
+}
 
 trait_set! {
   pub trait SendFn<T: MsgSet> = for<'a> FnMut(&'a [u8], ReqNot<T>) + DynClone + Send + 'static;
   pub trait ReqFn<T: MsgSet> =
-    FnMut(RequestHandle<T>) -> ReplyToken + DynClone + Send + Sync + 'static;
+    FnMut(RequestHandle<T>, <T::In as Channel>::Req) -> Receipt + DynClone + Send + Sync + 'static;
   pub trait NotifFn<T: MsgSet> =
     for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + DynClone + Send + Sync + 'static;
 }
@@ -24,28 +33,38 @@ fn get_id(message: &[u8]) -> (u64, &[u8]) {
   (u64::from_be_bytes(message[..8].to_vec().try_into().unwrap()), &message[8..])
 }
 
-pub struct RequestHandle<T: MsgSet> {
+pub trait ReqHandlish {
-  id: u64,
+  fn defer_drop(&self, val: impl Any + 'static);
-  message: <T::In as Channel>::Req,
+}
-  parent: ReqNot<T>,
+
+#[derive(destructure)]
+pub struct RequestHandle<MS: MsgSet> {
+  defer_drop: RefCell<Vec<Box<dyn Any>>>,
   fulfilled: AtomicBool,
+  id: u64,
+  parent: ReqNot<MS>,
 }
 impl<MS: MsgSet + 'static> RequestHandle<MS> {
+  fn new(parent: ReqNot<MS>, id: u64) -> Self {
+    Self { defer_drop: RefCell::default(), fulfilled: false.into(), parent, id }
+  }
   pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
-  pub fn req(&self) -> &<MS::In as Channel>::Req { &self.message }
+  pub fn handle<U: Request>(&self, _: &U, rep: &U::Response) -> Receipt { self.respond(rep) }
-  fn respond(&self, response: &impl Encode) -> ReplyToken {
+  pub fn will_handle_as<U: Request>(&self, _: &U) -> ReqTypToken<U> { ReqTypToken(PhantomData) }
+  pub fn handle_as<U: Request>(&self, _: ReqTypToken<U>, rep: &U::Response) -> Receipt {
+    self.respond(rep)
+  }
+  pub fn respond(&self, response: &impl Encode) -> Receipt {
     assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id);
     let mut buf = (!self.id).to_be_bytes().to_vec();
     response.encode(&mut buf);
     let mut send = clone_box(&*self.reqnot().0.lock().unwrap().send);
     (send)(&buf, self.parent.clone());
-    ReplyToken
+    Receipt
   }
-  pub fn handle<T: Request>(&self, _: &T, rep: &T::Response) -> ReplyToken { self.respond(rep) }
-  pub fn will_handle_as<T: Request>(&self, _: &T) -> ReqTypToken<T> { ReqTypToken(PhantomData) }
-  pub fn handle_as<T: Request>(&self, _token: ReqTypToken<T>, rep: &T::Response) -> ReplyToken {
-    self.respond(rep)
 }
+impl<MS: MsgSet> ReqHandlish for RequestHandle<MS> {
+  fn defer_drop(&self, val: impl Any) { self.defer_drop.borrow_mut().push(Box::new(val)) }
 }
 impl<MS: MsgSet> Drop for RequestHandle<MS> {
   fn drop(&mut self) {
@@ -56,10 +75,6 @@ impl<MS: MsgSet> Drop for RequestHandle<MS> {
 
 pub struct ReqTypToken<T>(PhantomData<T>);
 
-pub fn respond_with<R: Request>(r: &R, f: impl FnOnce(&R) -> R::Response) -> Vec<u8> {
-  r.respond(f(r))
-}
-
 pub struct ReqNotData<T: MsgSet> {
   id: u64,
   send: Box<dyn SendFn<T>>,
@@ -104,8 +119,11 @@ impl<T: MsgSet> ReqNot<T> {
       let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
       let mut req = clone_box(&*g.req);
       mem::drop(g);
-      let handle = RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() };
+      let rn = self.clone();
-      thread::Builder::new().name(format!("request {id}")).spawn(move || req(handle)).unwrap();
+      thread::Builder::new()
+        .name(format!("request {id}"))
+        .spawn(move || req(RequestHandle::new(rn, id), message))
+        .unwrap();
     }
   }
 
@@ -208,12 +226,12 @@ mod test {
     let receiver = ReqNot::<TestMsgSet>::new(
       |_, _| panic!("Should not send anything"),
       clone!(received; move |notif, _| *received.lock().unwrap() = Some(notif)),
-      |_| panic!("Not receiving a request"),
+      |_, _| panic!("Not receiving a request"),
     );
     let sender = ReqNot::<TestMsgSet>::new(
       clone!(receiver; move |d, _| receiver.receive(d.to_vec())),
       |_, _| panic!("Should not receive notif"),
-      |_| panic!("Should not receive request"),
+      |_, _| panic!("Should not receive request"),
     );
     sender.notify(3);
     assert_eq!(*received.lock().unwrap(), Some(3));
@@ -230,7 +248,7 @@ mod test {
         move |d, _| receiver.lock().unwrap().as_ref().unwrap().receive(d.to_vec())
       },
       |_, _| panic!("Should not receive notif"),
-      |_| panic!("Should not receive request"),
+      |_, _| panic!("Should not receive request"),
     ));
     *receiver.lock().unwrap() = Some(ReqNot::new(
       {
@@ -238,9 +256,9 @@ mod test {
         move |d, _| sender.receive(d.to_vec())
       },
       |_, _| panic!("Not receiving notifs"),
-      |req| {
+      |hand, req| {
-        assert_eq!(req.req(), &TestReq(5));
+        assert_eq!(req, TestReq(5));
-        req.respond(&6u8)
+        hand.respond(&6u8)
       },
     ));
     let response = sender.request(TestReq(5));

orchid-base/src/tree.rs

@@ -1,6 +1,6 @@
 use std::borrow::Borrow;
 use std::cell::RefCell;
-use std::fmt::{self, Display, Write};
+use std::fmt::{self, Debug, Display};
 use std::iter;
 use std::marker::PhantomData;
 use std::ops::Range;
@@ -8,26 +8,32 @@ use std::sync::Arc;
 
 use itertools::Itertools;
 use never::Never;
+use orchid_api::Placeholder;
+use ordered_float::NotNan;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::error::OrcErr;
+use crate::error::OrcErrv;
 use crate::interner::{deintern, Tok};
-use crate::name::{NameLike, VName};
+use crate::name::PathSlice;
+use crate::parse::Snippet;
 use crate::tokens::PARENS;
 
+pub use api::PhKind as PhKind;
+
 trait_set! {
-  pub trait RecurCB<'a, A: AtomInTok, X> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
+  pub trait RecurCB<'a, A: AtomTok, X: ExtraTok> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
+  pub trait ExtraTok = Display + Clone + fmt::Debug;
 }
 
-pub fn recur<'a, A: AtomInTok, X>(
+pub fn recur<'a, A: AtomTok, X: ExtraTok>(
   tt: TokTree<'a, A, X>,
   f: &impl Fn(TokTree<'a, A, X>, &dyn RecurCB<'a, A, X>) -> TokTree<'a, A, X>,
 ) -> TokTree<'a, A, X> {
   f(tt, &|TokTree { range, tok }| {
     let tok = match tok {
       tok @ (Token::Atom(_) | Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::NS) => tok,
-      tok @ (Token::Name(_) | Token::Slot(_) | Token::X(_)) => tok,
+      tok @ (Token::Name(_) | Token::Slot(_) | Token::X(_) | Token::Ph(_) | Token::Macro(_)) => tok,
       Token::LambdaHead(arg) =>
         Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
       Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
@@ -36,46 +42,48 @@ pub fn recur<'a, A: AtomInTok, X>(
   })
 }
 
-pub trait AtomInTok: Display + Clone {
+pub trait AtomTok: fmt::Display + Clone + fmt::Debug {
   type Context: ?Sized;
   fn from_api(atom: &api::Atom, pos: Range<u32>, ctx: &mut Self::Context) -> Self;
   fn to_api(&self) -> api::Atom;
 }
-impl AtomInTok for Never {
+impl AtomTok for Never {
   type Context = Never;
   fn from_api(_: &api::Atom, _: Range<u32>, _: &mut Self::Context) -> Self { panic!() }
   fn to_api(&self) -> orchid_api::Atom { match *self {} }
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
-pub struct TreeHandle<'a>(api::TreeTicket, PhantomData<&'a ()>);
+pub struct TokHandle<'a>(api::TreeTicket, PhantomData<&'a ()>);
-impl TreeHandle<'static> {
+impl TokHandle<'static> {
-  pub fn new(tt: api::TreeTicket) -> Self { TreeHandle(tt, PhantomData) }
+  pub fn new(tt: api::TreeTicket) -> Self { TokHandle(tt, PhantomData) }
 }
-impl<'a> TreeHandle<'a> {
+impl<'a> TokHandle<'a> {
   pub fn ticket(self) -> api::TreeTicket { self.0 }
 }
-impl<'a> Display for TreeHandle<'a> {
+impl<'a> Display for TokHandle<'a> {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Handle({})", self.0.0) }
 }
 
 #[derive(Clone, Debug)]
-pub struct TokTree<'a, A: AtomInTok, X> {
+pub struct TokTree<'a, A: AtomTok, X: ExtraTok> {
   pub tok: Token<'a, A, X>,
   pub range: Range<u32>,
 }
-impl<'a, A: AtomInTok, X> TokTree<'a, A, X> {
+impl<'a, A: AtomTok, X: ExtraTok> TokTree<'a, A, X> {
   pub fn from_api(tt: &api::TokenTree, ctx: &mut A::Context) -> Self {
     let tok = match &tt.token {
       api::Token::Atom(a) => Token::Atom(A::from_api(a, tt.range.clone(), ctx)),
       api::Token::BR => Token::BR,
       api::Token::NS => Token::NS,
-      api::Token::Bottom(e) => Token::Bottom(e.iter().map(OrcErr::from_api).collect()),
+      api::Token::Bottom(e) => Token::Bottom(OrcErrv::from_api(e)),
-      api::Token::Lambda(arg) => Token::LambdaHead(ttv_from_api(arg, ctx)),
+      api::Token::LambdaHead(arg) => Token::LambdaHead(ttv_from_api(arg, ctx)),
       api::Token::Name(name) => Token::Name(deintern(*name)),
-      api::Token::S(par, b) => Token::S(par.clone(), ttv_from_api(b, ctx)),
+      api::Token::S(par, b) => Token::S(*par, ttv_from_api(b, ctx)),
       api::Token::Comment(c) => Token::Comment(c.clone()),
-      api::Token::Slot(id) => Token::Slot(TreeHandle::new(*id)),
+      api::Token::Slot(id) => Token::Slot(TokHandle::new(*id)),
+      api::Token::Ph(ph) => Token::Ph(Ph {name: deintern(ph.name), kind: ph.kind }),
+      api::Token::Macro(prio) => Token::Macro(*prio)
     };
     Self { range: tt.range.clone(), tok }
   }
@@ -88,66 +96,110 @@ impl<'a, A: AtomInTok, X> TokTree<'a, A, X> {
       Token::Atom(a) => api::Token::Atom(a.to_api()),
       Token::BR => api::Token::BR,
       Token::NS => api::Token::NS,
-      Token::Bottom(e) => api::Token::Bottom(e.iter().map(OrcErr::to_api).collect()),
+      Token::Bottom(e) => api::Token::Bottom(e.to_api()),
       Token::Comment(c) => api::Token::Comment(c.clone()),
-      Token::LambdaHead(arg) =>
+      Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_to_api(arg, do_extra)),
-        api::Token::Lambda(arg.iter().map(|t| t.to_api(do_extra)).collect_vec()),
       Token::Name(n) => api::Token::Name(n.marker()),
       Token::Slot(tt) => api::Token::Slot(tt.ticket()),
-      Token::S(p, b) => api::Token::S(p.clone(), b.iter().map(|t| t.to_api(do_extra)).collect()),
+      Token::S(p, b) => api::Token::S(*p, ttv_to_api(b, do_extra)),
+      Token::Ph(Ph { name, kind }) => api::Token::Ph(Placeholder { name: name.marker(), kind: *kind }),
       Token::X(x) => return do_extra(x, self.range.clone()),
+      Token::Macro(prio) => api::Token::Macro(*prio),
     };
     api::TokenTree { range: self.range.clone(), token }
   }
+
+  pub fn into_api(
+    self,
+    do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
+  ) -> api::TokenTree {
+    let token = match self.tok {
+      Token::Atom(a) => api::Token::Atom(a.to_api()),
+      Token::BR => api::Token::BR,
+      Token::NS => api::Token::NS,
+      Token::Bottom(e) => api::Token::Bottom(e.to_api()),
+      Token::Comment(c) => api::Token::Comment(c.clone()),
+      Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra)),
+      Token::Name(n) => api::Token::Name(n.marker()),
+      Token::Slot(tt) => api::Token::Slot(tt.ticket()),
+      Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra)),
+      Token::Ph(Ph { kind, name }) => api::Token::Ph(Placeholder { name: name.marker(), kind }),
+      Token::X(x) => return do_extra(x, self.range.clone()),
+      Token::Macro(prio) => api::Token::Macro(prio),
+    };
+    api::TokenTree { range: self.range.clone(), token }
   }
-impl<'a, A: AtomInTok + Display, X: Display> Display for TokTree<'a, A, X> {
+
+  pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
+  pub fn as_name(&self) -> Option<Tok<String>> {
+    if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
+  }
+  pub fn as_s(&self, par: Paren) -> Option<Snippet<'_, 'a, A, X>> {
+    self.tok.as_s(par).map(|slc| Snippet::new(self, slc))
+  }
+  pub fn lambda(arg: Vec<Self>, mut body: Vec<Self>) -> Self {
+    let arg_range = ttv_range(&arg);
+    let s_range = arg_range.start..body.last().expect("Lambda with empty body!").range.end;
+    body.insert(0, Token::LambdaHead(arg).at(arg_range));
+    Token::S(Paren::Round, body).at(s_range)
+  }
+}
+
+impl<'a, A: AtomTok, X: ExtraTok> Display for TokTree<'a, A, X> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
 }
 
-pub fn ttv_from_api<A: AtomInTok, X>(
+pub fn ttv_from_api<A: AtomTok, X: ExtraTok>(
   tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
   ctx: &mut A::Context,
 ) -> Vec<TokTree<'static, A, X>> {
   tokv.into_iter().map(|t| TokTree::<A, X>::from_api(t.borrow(), ctx)).collect()
 }
 
-pub fn ttv_to_api<'a, A: AtomInTok, X>(
+pub fn ttv_to_api<'a, A: AtomTok, X: ExtraTok>(
   tokv: impl IntoIterator<Item: Borrow<TokTree<'a, A, X>>>,
   do_extra: &mut impl FnMut(&X, Range<u32>) -> api::TokenTree,
 ) -> Vec<api::TokenTree> {
-  tokv
+  tokv.into_iter().map(|tok| Borrow::<TokTree<A, X>>::borrow(&tok).to_api(do_extra)).collect_vec()
-    .into_iter()
-    .map(|tok| {
-      let tt: &TokTree<A, X> = tok.borrow();
-      tt.to_api(do_extra)
-    })
-    .collect_vec()
 }
 
-pub fn vname_tv<'a: 'b, 'b, A: AtomInTok + 'a, X: 'a>(
+pub fn ttv_into_api<'a, A: AtomTok, X: ExtraTok>(
-  name: &'b VName,
+  tokv: impl IntoIterator<Item = TokTree<'a, A, X>>,
-  ran: Range<u32>,
+  do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
+) -> Vec<api::TokenTree> {
+  tokv.into_iter().map(|t| t.into_api(do_extra)).collect_vec()
+}
+
+/// This takes a position and not a range because it assigns the range to
+/// multiple leaf tokens, which is only valid if it's a zero-width range
+pub fn vname_tv<'a: 'b, 'b, A: AtomTok + 'a, X: ExtraTok + 'a>(
+  name: &'b PathSlice,
+  pos: u32,
 ) -> impl Iterator<Item = TokTree<'a, A, X>> + 'b {
-  let (head, tail) = name.split_first();
+  let (head, tail) = name.split_first().expect("Empty vname");
-  iter::once(Token::Name(head))
+  iter::once(Token::Name(head.clone()))
-    .chain(tail.iter().flat_map(|t| [Token::NS, Token::Name(t)]))
+    .chain(tail.iter().flat_map(|t| [Token::NS, Token::Name(t.clone())]))
-    .map(move |t| t.at(ran.clone()))
+    .map(move |t| t.at(pos..pos))
 }
 
-pub fn wrap_tokv<'a, A: AtomInTok + 'a, X: 'a>(
+pub fn wrap_tokv<'a, A: AtomTok, X: ExtraTok>(
-  items: Vec<TokTree<'a, A, X>>,
+  items: impl IntoIterator<Item = TokTree<'a, A, X>>
-  range: Range<u32>,
 ) -> TokTree<'a, A, X> {
-  match items.len() {
+  let items_v = items.into_iter().collect_vec();
-    1 => items.into_iter().next().unwrap(),
+  match items_v.len() {
-    _ => Token::S(api::Paren::Round, items).at(range),
+    0 => panic!("A tokv with no elements is illegal"),
+    1 => items_v.into_iter().next().unwrap(),
+    _ => {
+      let range = items_v.first().unwrap().range.start..items_v.last().unwrap().range.end;
+      Token::S(api::Paren::Round, items_v).at(range)
+    },
   }
 }
 
 pub use api::Paren;
 
 #[derive(Clone, Debug)]
-pub enum Token<'a, A: AtomInTok, X> {
+pub enum Token<'a, A: AtomTok, X: ExtraTok> {
   Comment(Arc<String>),
   LambdaHead(Vec<TokTree<'a, A, X>>),
   Name(Tok<String>),
@@ -155,14 +207,25 @@ pub enum Token<'a, A: AtomInTok, X> {
   BR,
   S(Paren, Vec<TokTree<'a, A, X>>),
   Atom(A),
-  Bottom(Vec<OrcErr>),
+  Bottom(OrcErrv),
-  Slot(TreeHandle<'a>),
+  Slot(TokHandle<'a>),
   X(X),
+  Ph(Ph),
+  Macro(Option<NotNan<f64>>),
 }
-impl<'a, A: AtomInTok, X> Token<'a, A, X> {
+impl<'a, A: AtomTok, X: ExtraTok> Token<'a, A, X> {
   pub fn at(self, range: Range<u32>) -> TokTree<'a, A, X> { TokTree { range, tok: self } }
+  pub fn is_kw(&self, tk: Tok<String>) -> bool {
+    matches!(self, Token::Name(n) if *n == tk)
   }
-impl<'a, A: AtomInTok + Display, X: Display> Display for Token<'a, A, X> {
+  pub fn as_s(&self, par: Paren) -> Option<&[TokTree<'a, A, X>]> {
+    match self {
+      Self::S(p, b) if *p == par => Some(b),
+      _ => None,
+    }
+  }
+}
+impl<'a, A: AtomTok, X: ExtraTok> Display for Token<'a, A, X> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     thread_local! {
       static PAREN_LEVEL: RefCell<usize> = 0.into();
@@ -177,29 +240,43 @@ impl<'a, A: AtomInTok + Display, X: Display> Display for Token<'a, A, X> {
     match self {
       Self::Atom(a) => f.write_str(&indent(&format!("{a} "), get_indent(), false)),
       Self::BR => write!(f, "\n{}", "  ".repeat(get_indent())),
-      Self::Bottom(err) => write!(
+      Self::Bottom(err) if err.len() == 1 => write!(f, "Bottom({}) ", err.one().unwrap()),
-        f,
+      Self::Bottom(err) => {
-        "Botttom({})",
+        write!(f, "Botttom(\n{}) ", indent(&err.to_string(), get_indent() + 1, true))
-        err.iter().map(|e| format!("{}: {}", e.description, e.message)).join(", ")
+      },
-      ),
       Self::Comment(c) => write!(f, "--[{c}]-- "),
       Self::LambdaHead(arg) => with_indent(|| write!(f, "\\ {} . ", ttv_fmt(arg))),
       Self::NS => f.write_str(":: "),
-      Self::Name(n) => f.write_str(n),
+      Self::Name(n) => write!(f, "{n} "),
       Self::Slot(th) => write!(f, "{th} "),
+      Self::Ph(Ph { kind, name }) => match &kind {
+        PhKind::Scalar => write!(f, "${name}"),
+        PhKind::Vector { at_least_one, priority } => {
+          if *at_least_one { write!(f, ".")? }
+          write!(f, "..${name}")?;
+          if 0 < *priority { write!(f, "{priority}") } else { Ok(()) }
+        }
+      }
       Self::S(p, b) => {
         let (lp, rp, _) = PARENS.iter().find(|(_, _, par)| par == p).unwrap();
-        f.write_char(*lp)?;
+        write!(f, "{lp} ")?;
         with_indent(|| f.write_str(&ttv_fmt(b)))?;
-        f.write_char(*rp)
+        write!(f, "{rp} ")
       },
       Self::X(x) => write!(f, "{x} "),
+      Self::Macro(None) => write!(f, "macro "),
+      Self::Macro(Some(prio)) => write!(f, "macro({prio})"),
     }
   }
 }
 
-pub fn ttv_fmt<'a>(
+pub fn ttv_range(ttv: &[TokTree<'_, impl AtomTok, impl ExtraTok>]) -> Range<u32> {
-  ttv: impl IntoIterator<Item = &'a TokTree<'a, impl AtomInTok + 'a, impl Display + 'a>>,
+  assert!(!ttv.is_empty(), "Empty slice has no range");
+  ttv.first().unwrap().range.start..ttv.last().unwrap().range.end
+}
+
+pub fn ttv_fmt<'a: 'b, 'b>(
+  ttv: impl IntoIterator<Item = &'b TokTree<'a, impl AtomTok + 'b, impl ExtraTok + 'b>>,
 ) -> String {
   ttv.into_iter().join("")
 }
@@ -214,13 +291,23 @@ pub fn indent(s: &str, lvl: usize, first: bool) -> String {
   }
 }
 
+#[derive(Clone, Debug)]
+pub struct Ph {
+  pub name: Tok<String>,
+  pub kind: PhKind,
+}
+impl Ph {
+  pub fn from_api(api: &Placeholder) -> Self { Self { name: deintern(api.name), kind: api.kind } }
+  pub fn to_api(&self) -> Placeholder { Placeholder { name: self.name.marker(), kind: self.kind } }
+}
+
 #[cfg(test)]
 mod test {
   use super::*;
 
   #[test]
   fn test_covariance() {
-    fn _f<'a>(x: Token<'static, Never, ()>) -> Token<'a, Never, ()> { x }
+    fn _f<'a>(x: Token<'static, Never, Never>) -> Token<'a, Never, Never> { x }
   }
 
   #[test]

orchid-extension/src/atom.rs

@@ -3,44 +3,42 @@ use std::fmt;
 use std::io::{Read, Write};
 use std::marker::PhantomData;
 use std::ops::{Deref, Range};
-use std::sync::OnceLock;
+use std::sync::{Arc, OnceLock};
 
 use dyn_clone::{clone_box, DynClone};
-use never::Never;
+use orchid_api_traits::{enc_vec, Coding, Decode, Encode, Request};
-use orchid_api::ExprTicket;
-use orchid_api_traits::{enc_vec, Coding, Decode, Request};
 use orchid_base::error::{mk_err, OrcErr, OrcRes};
 use orchid_base::intern;
 use orchid_base::location::Pos;
+use orchid_base::name::Sym;
 use orchid_base::reqnot::Requester;
-use orchid_base::tree::AtomInTok;
+use orchid_base::tree::AtomTok;
 use trait_set::trait_set;
 
 use crate::api;
 // use crate::error::{ProjectError, ProjectResult};
-use crate::expr::{ExprHandle, GenClause, GenExpr, OwnedExpr};
+use crate::expr::{Expr, ExprData, ExprHandle, ExprKind};
 use crate::system::{atom_info_for, downcast_atom, DynSystemCard, SysCtx};
 
 pub trait AtomCard: 'static + Sized {
   type Data: Clone + Coding + Sized;
-  type Req: Coding;
 }
 
 pub trait AtomicVariant {}
 pub trait Atomic: 'static + Sized {
   type Variant: AtomicVariant;
   type Data: Clone + Coding + Sized;
-  type Req: Coding;
+  fn reg_reqs() -> MethodSet<Self>;
 }
 impl<A: Atomic> AtomCard for A {
   type Data = <Self as Atomic>::Data;
-  type Req = <Self as Atomic>::Req;
 }
 
 pub trait AtomicFeatures: Atomic {
   fn factory(self) -> AtomFactory;
   type Info: AtomDynfo;
-  const INFO: &'static Self::Info;
+  fn info() -> Self::Info;
+  fn dynfo() -> Box<dyn AtomDynfo>;
 }
 pub trait ToAtom {
   fn to_atom_factory(self) -> AtomFactory;
@@ -54,17 +52,18 @@ impl ToAtom for AtomFactory {
 pub trait AtomicFeaturesImpl<Variant: AtomicVariant> {
   fn _factory(self) -> AtomFactory;
   type _Info: AtomDynfo;
-  const _INFO: &'static Self::_Info;
+  fn _info() -> Self::_Info;
 }
-impl<A: Atomic + AtomicFeaturesImpl<A::Variant> + ?Sized> AtomicFeatures for A {
+impl<A: Atomic + AtomicFeaturesImpl<A::Variant>> AtomicFeatures for A {
   fn factory(self) -> AtomFactory { self._factory() }
   type Info = <Self as AtomicFeaturesImpl<A::Variant>>::_Info;
-  const INFO: &'static Self::Info = Self::_INFO;
+  fn info() -> Self::Info { Self::_info() }
+  fn dynfo() -> Box<dyn AtomDynfo> { Box::new(Self::info()) }
 }
 
 pub fn get_info<A: AtomCard>(
   sys: &(impl DynSystemCard + ?Sized),
-) -> (api::AtomId, &'static dyn AtomDynfo) {
+) -> (api::AtomId, Box<dyn AtomDynfo>) {
   atom_info_for(sys, TypeId::of::<A>()).unwrap_or_else(|| {
     panic!("Atom {} not associated with system {}", type_name::<A>(), sys.name())
   })
@@ -72,34 +71,47 @@ pub fn get_info<A: AtomCard>(
 
 #[derive(Clone)]
 pub struct ForeignAtom<'a> {
-  pub expr: Option<ExprHandle>,
+  pub expr: Option<Arc<ExprHandle>>,
-  pub char_marker: PhantomData<&'a ()>,
+  pub _life: PhantomData<&'a ()>,
   pub ctx: SysCtx,
   pub atom: api::Atom,
   pub pos: Pos,
 }
 impl<'a> ForeignAtom<'a> {
-  pub fn oex_opt(self) -> Option<OwnedExpr> {
+  pub fn oex_opt(self) -> Option<Expr> {
-    self.expr.map(|handle| {
+    let (handle, pos) = (self.expr.as_ref()?.clone(), self.pos.clone());
-      let gen_expr = GenExpr { pos: self.pos, clause: GenClause::Atom(handle.tk, self.atom) };
+    let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { _life: PhantomData, ..self }) };
-      OwnedExpr { handle, val: OnceLock::from(Box::new(gen_expr)) }
+    Some(Expr { handle: Some(handle), val: OnceLock::from(data) })
-    })
   }
 }
 impl ForeignAtom<'static> {
-  pub fn oex(self) -> OwnedExpr { self.oex_opt().unwrap() }
+  pub fn oex(self) -> Expr { self.oex_opt().unwrap() }
+  pub(crate) fn new(handle: Arc<ExprHandle>, atom: api::Atom, pos: Pos) -> Self {
+    ForeignAtom { _life: PhantomData, atom, ctx: handle.ctx.clone(), expr: Some(handle), pos }
+  }
+  pub fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
+    let rep = self.ctx.reqnot.request(api::Fwd(
+      self.atom.clone(),
+      Sym::parse(M::NAME).unwrap().tok().marker(),
+      enc_vec(&m)
+    ))?;
+    Some(M::Response::decode(&mut &rep[..]))
+  }
 }
 impl<'a> fmt::Display for ForeignAtom<'a> {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     write!(f, "{}::{:?}", if self.expr.is_some() { "Clause" } else { "Tok" }, self.atom)
   }
 }
-impl<'a> AtomInTok for ForeignAtom<'a> {
+impl<'a> fmt::Debug for ForeignAtom<'a> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ForeignAtom({self})") }
+}
+impl<'a> AtomTok for ForeignAtom<'a> {
   type Context = SysCtx;
   fn from_api(atom: &api::Atom, pos: Range<u32>, ctx: &mut Self::Context) -> Self {
     Self {
       atom: atom.clone(),
-      char_marker: PhantomData,
+      _life: PhantomData,
       ctx: ctx.clone(),
       expr: None,
       pos: Pos::Range(pos),
@@ -108,7 +120,7 @@ impl<'a> AtomInTok for ForeignAtom<'a> {
   fn to_api(&self) -> orchid_api::Atom { self.atom.clone() }
 }
 
-pub struct NotTypAtom(pub Pos, pub OwnedExpr, pub &'static dyn AtomDynfo);
+pub struct NotTypAtom(pub Pos, pub Expr, pub Box<dyn AtomDynfo>);
 impl NotTypAtom {
   pub fn mk_err(&self) -> OrcErr {
     mk_err(
@@ -119,26 +131,86 @@ impl NotTypAtom {
   }
 }
 
+pub trait AtomMethod: Request {
+  const NAME: &str;
+}
+pub trait Supports<M: AtomMethod>: AtomCard {
+  fn handle(&self, ctx: SysCtx, req: M) -> <M as Request>::Response;
+}
+
+trait_set! {
+  trait AtomReqCb<A> = Fn(&A, SysCtx, &mut dyn Read, &mut dyn Write) + Send + Sync
+}
+
+pub struct AtomReqHandler<A: AtomCard> {
+  key: Sym,
+  cb: Box<dyn AtomReqCb<A>>,
+}
+
+pub struct MethodSet<A: AtomCard> {
+  handlers: Vec<AtomReqHandler<A>>,
+}
+impl<A: AtomCard> MethodSet<A> {
+  pub fn new() -> Self { Self{ handlers: vec![] } }
+
+  pub fn handle<M: AtomMethod>(mut self) -> Self where A: Supports<M> {
+    self.handlers.push(AtomReqHandler {
+      key: Sym::parse(M::NAME).expect("AtomMethod::NAME cannoot be empty"),
+      cb: Box::new(move |
+        a: &A,
+        ctx: SysCtx,
+        req: &mut dyn Read,
+        rep: &mut dyn Write
+      | {
+        Supports::<M>::handle(a, ctx, M::decode(req)).encode(rep);
+      })
+    });
+    self
+  }
+
+  pub(crate) fn dispatch(
+    &self, atom: &A, ctx: SysCtx, key: Sym, req: &mut dyn Read, rep: &mut dyn Write
+  ) -> bool {
+    match self.handlers.iter().find(|h| h.key == key) {
+      None => false,
+      Some(handler) => {
+        (handler.cb)(atom, ctx, req, rep);
+        true
+      },
+    }
+  }
+}
+
+impl<A: AtomCard> Default for MethodSet<A> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 #[derive(Clone)]
 pub struct TypAtom<'a, A: AtomicFeatures> {
   pub data: ForeignAtom<'a>,
   pub value: A::Data,
 }
 impl<A: AtomicFeatures> TypAtom<'static, A> {
-  pub fn downcast(expr: ExprHandle) -> Result<Self, NotTypAtom> {
+  pub fn downcast(expr: Arc<ExprHandle>) -> Result<Self, NotTypAtom> {
-    match OwnedExpr::new(expr).foreign_atom() {
+    match Expr::new(expr).foreign_atom() {
-      Err(oe) => Err(NotTypAtom(oe.get_data().pos.clone(), oe, A::INFO)),
+      Err(oe) => Err(NotTypAtom(oe.get_data().pos.clone(), oe, Box::new(A::info()))),
       Ok(atm) => match downcast_atom::<A>(atm) {
-        Err(fa) => Err(NotTypAtom(fa.pos.clone(), fa.oex(), A::INFO)),
+        Err(fa) => Err(NotTypAtom(fa.pos.clone(), fa.oex(), Box::new(A::info()))),
         Ok(tatom) => Ok(tatom),
       },
     }
   }
 }
 impl<'a, A: AtomicFeatures> TypAtom<'a, A> {
-  pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
+  pub fn request<M: AtomMethod>(&self, req: M) -> M::Response where A: Supports<M> {
-    R::Response::decode(
+    M::Response::decode(
-      &mut &self.data.ctx.reqnot.request(api::Fwd(self.data.atom.clone(), enc_vec(&req)))[..],
+      &mut &self.data.ctx.reqnot.request(api::Fwd(
+        self.data.atom.clone(),
+        Sym::parse(M::NAME).unwrap().tok().marker(),
+        enc_vec(&req)
+      )).unwrap()[..]
     )
   }
 }
@@ -153,14 +225,13 @@ pub trait AtomDynfo: Send + Sync + 'static {
   fn tid(&self) -> TypeId;
   fn name(&self) -> &'static str;
   fn decode(&self, ctx: AtomCtx<'_>) -> Box<dyn Any>;
-  fn call(&self, ctx: AtomCtx<'_>, arg: api::ExprTicket) -> GenExpr;
+  fn call(&self, ctx: AtomCtx<'_>, arg: api::ExprTicket) -> Expr;
-  fn call_ref(&self, ctx: AtomCtx<'_>, arg: api::ExprTicket) -> GenExpr;
+  fn call_ref(&self, ctx: AtomCtx<'_>, arg: api::ExprTicket) -> Expr;
-  fn same(&self, ctx: AtomCtx<'_>, other: &api::Atom) -> bool;
   fn print(&self, ctx: AtomCtx<'_>) -> String;
-  fn handle_req(&self, ctx: AtomCtx<'_>, req: &mut dyn Read, rep: &mut dyn Write);
+  fn handle_req(&self, ctx: AtomCtx<'_>, key: Sym, req: &mut dyn Read, rep: &mut dyn Write) -> bool;
-  fn command(&self, ctx: AtomCtx<'_>) -> OrcRes<Option<GenExpr>>;
+  fn command(&self, ctx: AtomCtx<'_>) -> OrcRes<Option<Expr>>;
-  fn serialize(&self, ctx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<ExprTicket>;
+  fn serialize(&self, ctx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<api::ExprTicket>;
-  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[ExprTicket]) -> api::Atom;
+  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> api::Atom;
   fn drop(&self, ctx: AtomCtx<'_>);
 }
 
@@ -177,6 +248,12 @@ impl AtomFactory {
 impl Clone for AtomFactory {
   fn clone(&self) -> Self { AtomFactory(clone_box(&*self.0)) }
 }
+impl fmt::Debug for AtomFactory {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "AtomFactory") }
+}
+impl fmt::Display for AtomFactory {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "AtomFactory") }
+}
 
 pub fn err_not_callable() -> OrcErr {
   mk_err(intern!(str: "This atom is not callable"), "Attempted to apply value as function", [])
@@ -185,26 +262,3 @@ pub fn err_not_callable() -> OrcErr {
 pub fn err_not_command() -> OrcErr {
   mk_err(intern!(str: "This atom is not a command"), "Settled on an inactionable value", [])
 }
-
-pub trait ReqPck<T: AtomCard + ?Sized>: Sized {
-  type W: Write + ?Sized;
-  fn unpack<'a>(self) -> (T::Req, &'a mut Self::W, SysCtx)
-  where Self: 'a;
-  fn never(self)
-  where T: AtomCard<Req = Never> {
-  }
-}
-
-pub(crate) struct RequestPack<'a, T: AtomCard + ?Sized, W: Write + ?Sized> {
-  pub req: T::Req,
-  pub write: &'a mut W,
-  pub sys: SysCtx,
-}
-
-impl<'a, T: AtomCard + ?Sized, W: Write + ?Sized> ReqPck<T> for RequestPack<'a, T, W> {
-  type W = W;
-  fn unpack<'b>(self) -> (<T as AtomCard>::Req, &'b mut Self::W, SysCtx)
-  where 'a: 'b {
-    (self.req, self.write, self.sys)
-  }
-}

orchid-extension/src/atom_owned.rs

@@ -1,19 +1,19 @@
 use std::any::{type_name, Any, TypeId};
 use std::borrow::Cow;
 use std::io::{Read, Write};
-use std::marker::PhantomData;
+use std::sync::Arc;
 
 use itertools::Itertools;
 use orchid_api_traits::{enc_vec, Decode, Encode};
 use orchid_base::error::OrcRes;
 use orchid_base::id_store::{IdRecord, IdStore};
+use orchid_base::name::Sym;
 
 use crate::api;
 use crate::atom::{
-  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic,
+  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, MethodSet, Atomic, AtomicFeaturesImpl, AtomicVariant,
-  AtomicFeaturesImpl, AtomicVariant, ReqPck, RequestPack,
 };
-use crate::expr::{bot, ExprHandle, GenExpr};
+use crate::expr::{bot, Expr, ExprHandle};
 use crate::system::SysCtx;
 
 pub struct OwnedVariant;
@@ -28,15 +28,17 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
       api::Atom { drop: Some(api::AtomId(rec.id())), data, owner: ctx.id }
     })
   }
+  fn _info() -> Self::_Info {
+    OwnedAtomDynfo(A::reg_reqs())
+  }
   type _Info = OwnedAtomDynfo<A>;
-  const _INFO: &'static Self::_Info = &OwnedAtomDynfo(PhantomData);
 }
 
 fn with_atom<U>(id: api::AtomId, f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>) -> U) -> U {
   f(OBJ_STORE.get(id.0).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0)))
 }
 
-pub struct OwnedAtomDynfo<T: OwnedAtom>(PhantomData<T>);
+pub struct OwnedAtomDynfo<T: OwnedAtom>(MethodSet<T>);
 impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
   fn print(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> String {
     with_atom(id.unwrap(), |a| a.dyn_print(ctx))
@@ -46,19 +48,18 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
   fn decode(&self, AtomCtx(data, ..): AtomCtx) -> Box<dyn Any> {
     Box::new(<T as AtomCard>::Data::decode(&mut &data[..]))
   }
-  fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
+  fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> Expr {
     with_atom(id.unwrap(), |a| a.remove().dyn_call(ctx, arg))
   }
-  fn call_ref(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
+  fn call_ref(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> Expr {
     with_atom(id.unwrap(), |a| a.dyn_call_ref(ctx, arg))
   }
-  fn same(&self, AtomCtx(_, id, ctx): AtomCtx, a2: &api::Atom) -> bool {
+  fn handle_req(&self, AtomCtx(_, id, ctx): AtomCtx, key: Sym, req: &mut dyn Read, rep: &mut dyn Write) -> bool {
-    with_atom(id.unwrap(), |a1| with_atom(a2.drop.unwrap(), |a2| a1.dyn_same(ctx, &**a2)))
+    with_atom(id.unwrap(), |a| {
+      self.0.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx, key, req, rep)
+    })
   }
-  fn handle_req(&self, AtomCtx(_, id, ctx): AtomCtx, req: &mut dyn Read, rep: &mut dyn Write) {
+  fn command(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> OrcRes<Option<Expr>> {
-    with_atom(id.unwrap(), |a| a.dyn_handle_req(ctx, req, rep))
-  }
-  fn command(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> OrcRes<Option<GenExpr>> {
     with_atom(id.unwrap(), |a| a.remove().dyn_command(ctx))
   }
   fn drop(&self, AtomCtx(_, id, ctx): AtomCtx) {
@@ -71,10 +72,13 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
   ) -> Vec<api::ExprTicket> {
     let id = id.unwrap();
     id.encode(write);
-    with_atom(id, |a| a.dyn_serialize(ctx, write)).into_iter().map(|t| t.into_tk()).collect_vec()
+    with_atom(id, |a| a.dyn_serialize(ctx, write))
+      .into_iter()
+      .map(|t| t.handle.unwrap().tk)
+      .collect_vec()
   }
   fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> orchid_api::Atom {
-    let refs = refs.iter().map(|tk| ExprHandle::from_args(ctx.clone(), *tk));
+    let refs = refs.iter().map(|tk| Expr::new(Arc::new(ExprHandle::from_args(ctx.clone(), *tk))));
     let obj = T::deserialize(DeserCtxImpl(data, &ctx), T::Refs::from_iter(refs));
     obj._factory().build(ctx)
   }
@@ -100,27 +104,25 @@ impl<'a> DeserializeCtx for DeserCtxImpl<'a> {
 }
 
 pub trait RefSet {
-  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self;
+  fn from_iter<I: Iterator<Item = Expr> + ExactSizeIterator>(refs: I) -> Self;
-  fn to_vec(self) -> Vec<ExprHandle>;
+  fn to_vec(self) -> Vec<Expr>;
 }
 
 impl RefSet for () {
-  fn to_vec(self) -> Vec<ExprHandle> { Vec::new() }
+  fn to_vec(self) -> Vec<Expr> { Vec::new() }
-  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self {
+  fn from_iter<I: Iterator<Item = Expr> + ExactSizeIterator>(refs: I) -> Self {
     assert_eq!(refs.len(), 0, "Expected no refs")
   }
 }
 
-impl RefSet for Vec<ExprHandle> {
+impl RefSet for Vec<Expr> {
-  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self {
+  fn from_iter<I: Iterator<Item = Expr> + ExactSizeIterator>(refs: I) -> Self { refs.collect_vec() }
-    refs.collect_vec()
+  fn to_vec(self) -> Vec<Expr> { self }
-  }
-  fn to_vec(self) -> Vec<ExprHandle> { self }
 }
 
-impl<const N: usize> RefSet for [ExprHandle; N] {
+impl<const N: usize> RefSet for [Expr; N] {
-  fn to_vec(self) -> Vec<ExprHandle> { self.into_iter().collect_vec() }
+  fn to_vec(self) -> Vec<Expr> { self.into_iter().collect_vec() }
-  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self {
+  fn from_iter<I: Iterator<Item = Expr> + ExactSizeIterator>(refs: I) -> Self {
     assert_eq!(refs.len(), N, "Wrong number of refs provided");
     refs.collect_vec().try_into().unwrap_or_else(|_: Vec<_>| unreachable!())
   }
@@ -131,22 +133,15 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone
   type Refs: RefSet;
   fn val(&self) -> Cow<'_, Self::Data>;
   #[allow(unused_variables)]
-  fn call_ref(&self, arg: ExprHandle) -> GenExpr { bot(err_not_callable()) }
+  fn call_ref(&self, arg: ExprHandle) -> Expr { bot([err_not_callable()]) }
-  fn call(self, arg: ExprHandle) -> GenExpr {
+  fn call(self, arg: ExprHandle) -> Expr {
     let ctx = arg.get_ctx();
     let gcl = self.call_ref(arg);
     self.free(ctx);
     gcl
   }
   #[allow(unused_variables)]
-  fn same(&self, ctx: SysCtx, other: &Self) -> bool {
+  fn command(self, ctx: SysCtx) -> OrcRes<Option<Expr>> { Err(err_not_command().into()) }
-    let tname = type_name::<Self>();
-    writeln!(ctx.logger, "Override OwnedAtom::same for {tname} if it can appear in macro input");
-    false
-  }
-  fn handle_req(&self, pck: impl ReqPck<Self>);
-  #[allow(unused_variables)]
-  fn command(self, ctx: SysCtx) -> OrcRes<Option<GenExpr>> { Err(vec![err_not_command()]) }
   #[allow(unused_variables)]
   fn free(self, ctx: SysCtx) {}
   #[allow(unused_variables)]
@@ -159,41 +154,27 @@ pub trait DynOwnedAtom: Send + Sync + 'static {
   fn atom_tid(&self) -> TypeId;
   fn as_any_ref(&self) -> &dyn Any;
   fn encode(&self, buffer: &mut dyn Write);
-  fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr;
+  fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> Expr;
-  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr;
+  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket) -> Expr;
-  fn dyn_same(&self, ctx: SysCtx, other: &dyn DynOwnedAtom) -> bool;
+  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<Expr>>;
-  fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write);
-  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<GenExpr>>;
   fn dyn_free(self: Box<Self>, ctx: SysCtx);
   fn dyn_print(&self, ctx: SysCtx) -> String;
-  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<ExprHandle>;
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<Expr>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
   fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
   fn as_any_ref(&self) -> &dyn Any { self }
   fn encode(&self, buffer: &mut dyn Write) { self.val().as_ref().encode(buffer) }
-  fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr {
+  fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> Expr {
     self.call_ref(ExprHandle::from_args(ctx, arg))
   }
-  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr {
+  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket) -> Expr {
     self.call(ExprHandle::from_args(ctx, arg))
   }
-  fn dyn_same(&self, ctx: SysCtx, other: &dyn DynOwnedAtom) -> bool {
+  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<Expr>> { self.command(ctx) }
-    if TypeId::of::<Self>() != other.as_any_ref().type_id() {
-      return false;
-    }
-    let other_self = other.as_any_ref().downcast_ref().expect("The type_ids are the same");
-    self.same(ctx, other_self)
-  }
-  fn dyn_handle_req(&self, sys: SysCtx, req: &mut dyn Read, write: &mut dyn Write) {
-    let pack =
-      RequestPack::<T, dyn Write> { req: <Self as AtomCard>::Req::decode(req), write, sys };
-    self.handle_req(pack)
-  }
-  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<GenExpr>> { self.command(ctx) }
   fn dyn_free(self: Box<Self>, ctx: SysCtx) { self.free(ctx) }
   fn dyn_print(&self, ctx: SysCtx) -> String { self.print(ctx) }
-  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<ExprHandle> {
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<Expr> {
     self.serialize(ctx, sink).to_vec()
   }
 }

orchid-extension/src/atom_thin.rs

@@ -1,17 +1,15 @@
 use std::any::{type_name, Any, TypeId};
 use std::io::Write;
-use std::marker::PhantomData;
 
-use orchid_api::ExprTicket;
+use orchid_api_traits::{enc_vec, Coding};
-use orchid_api_traits::{enc_vec, Coding, Decode};
 use orchid_base::error::OrcRes;
+use orchid_base::name::Sym;
 
 use crate::api;
 use crate::atom::{
-  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic,
+  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, MethodSet, Atomic, AtomicFeaturesImpl, AtomicVariant
-  AtomicFeaturesImpl, AtomicVariant, ReqPck, RequestPack,
 };
-use crate::expr::{bot, ExprHandle, GenExpr};
+use crate::expr::{bot, Expr, ExprHandle};
 use crate::system::SysCtx;
 
 pub struct ThinVariant;
@@ -25,11 +23,13 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
       api::Atom { drop: None, data: buf, owner: ctx.id }
     })
   }
+  fn _info() -> Self::_Info {
+    ThinAtomDynfo(Self::reg_reqs())
+  }
   type _Info = ThinAtomDynfo<Self>;
-  const _INFO: &'static Self::_Info = &ThinAtomDynfo(PhantomData);
 }
 
-pub struct ThinAtomDynfo<T: ThinAtom>(PhantomData<T>);
+pub struct ThinAtomDynfo<T: ThinAtom>(MethodSet<T>);
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
   fn print(&self, AtomCtx(buf, _, ctx): AtomCtx<'_>) -> String {
     T::decode(&mut &buf[..]).print(ctx)
@@ -37,32 +37,29 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
   fn tid(&self) -> TypeId { TypeId::of::<T>() }
   fn name(&self) -> &'static str { type_name::<T>() }
   fn decode(&self, AtomCtx(buf, ..): AtomCtx) -> Box<dyn Any> { Box::new(T::decode(&mut &buf[..])) }
-  fn call(&self, AtomCtx(buf, _, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
+  fn call(&self, AtomCtx(buf, _, ctx): AtomCtx, arg: api::ExprTicket) -> Expr {
     T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg))
   }
-  fn call_ref(&self, AtomCtx(buf, _, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
+  fn call_ref(&self, AtomCtx(buf, _, ctx): AtomCtx, arg: api::ExprTicket) -> Expr {
     T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg))
   }
   fn handle_req(
     &self,
     AtomCtx(buf, _, sys): AtomCtx,
+    key: Sym,
     req: &mut dyn std::io::Read,
-    write: &mut dyn Write,
+    rep: &mut dyn Write,
-  ) {
+  ) -> bool {
-    let pack = RequestPack::<T, dyn Write> { req: Decode::decode(req), write, sys };
+    self.0.dispatch(&T::decode(&mut &buf[..]), sys, key, req, rep)
-    T::decode(&mut &buf[..]).handle_req(pack)
   }
-  fn same(&self, AtomCtx(buf, _, ctx): AtomCtx, a2: &api::Atom) -> bool {
+  fn command(&self, AtomCtx(buf, _, ctx): AtomCtx<'_>) -> OrcRes<Option<Expr>> {
-    T::decode(&mut &buf[..]).same(ctx, &T::decode(&mut &a2.data[8..]))
-  }
-  fn command(&self, AtomCtx(buf, _, ctx): AtomCtx<'_>) -> OrcRes<Option<GenExpr>> {
     T::decode(&mut &buf[..]).command(ctx)
   }
-  fn serialize(&self, AtomCtx(buf, ..): AtomCtx<'_>, write: &mut dyn Write) -> Vec<ExprTicket> {
+  fn serialize(&self, actx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<api::ExprTicket> {
-    T::decode(&mut &buf[..]).encode(write);
+    T::decode(&mut &actx.0[..]).encode(write);
     Vec::new()
   }
-  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[ExprTicket]) -> api::Atom {
+  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> api::Atom {
     assert!(refs.is_empty(), "Refs found when deserializing thin atom");
     T::decode(&mut &data[..])._factory().build(ctx)
   }
@@ -76,16 +73,9 @@ pub trait ThinAtom:
   AtomCard<Data = Self> + Atomic<Variant = ThinVariant> + Coding + Send + Sync + 'static
 {
   #[allow(unused_variables)]
-  fn call(&self, arg: ExprHandle) -> GenExpr { bot(err_not_callable()) }
+  fn call(&self, arg: ExprHandle) -> Expr { bot([err_not_callable()]) }
   #[allow(unused_variables)]
-  fn same(&self, ctx: SysCtx, other: &Self) -> bool {
+  fn command(&self, ctx: SysCtx) -> OrcRes<Option<Expr>> { Err(err_not_command().into()) }
-    let tname = type_name::<Self>();
-    writeln!(ctx.logger, "Override ThinAtom::same for {tname} if it can appear in macro input");
-    false
-  }
-  fn handle_req(&self, pck: impl ReqPck<Self>);
-  #[allow(unused_variables)]
-  fn command(&self, ctx: SysCtx) -> OrcRes<Option<GenExpr>> { Err(vec![err_not_command()]) }
   #[allow(unused_variables)]
   fn print(&self, ctx: SysCtx) -> String { format!("ThinAtom({})", type_name::<Self>()) }
 }

orchid-extension/src/conv.rs

@@ -3,19 +3,19 @@ use orchid_base::intern;
 use orchid_base::location::Pos;
 
 use crate::atom::{AtomicFeatures, ToAtom, TypAtom};
-use crate::expr::{atom, botv, ExprHandle, GenExpr, OwnedExpr};
+use crate::expr::{atom, bot, Expr};
 use crate::system::downcast_atom;
 
 pub trait TryFromExpr: Sized {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self>;
+  fn try_from_expr(expr: Expr) -> OrcRes<Self>;
 }
 
-impl TryFromExpr for OwnedExpr {
+impl TryFromExpr for Expr {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> { Ok(OwnedExpr::new(expr)) }
+  fn try_from_expr(expr: Expr) -> OrcRes<Self> { Ok(expr) }
 }
 
 impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
+  fn try_from_expr(expr: Expr) -> OrcRes<Self> {
     Ok((T::try_from_expr(expr.clone())?, U::try_from_expr(expr)?))
   }
 }
@@ -29,31 +29,30 @@ fn err_type(pos: Pos) -> OrcErr {
 }
 
 impl<'a, A: AtomicFeatures> TryFromExpr for TypAtom<'a, A> {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
+  fn try_from_expr(expr: Expr) -> OrcRes<Self> {
-    OwnedExpr::new(expr)
+    (expr.foreign_atom())
-      .foreign_atom()
+      .map_err(|ex| err_not_atom(ex.pos.clone()).into())
-      .map_err(|ex| vec![err_not_atom(ex.pos.clone())])
+      .and_then(|f| downcast_atom(f).map_err(|f| err_type(f.pos).into()))
-      .and_then(|f| downcast_atom(f).map_err(|f| vec![err_type(f.pos)]))
   }
 }
 
 pub trait ToExpr {
-  fn to_expr(self) -> GenExpr;
+  fn to_expr(self) -> Expr;
 }
 
-impl ToExpr for GenExpr {
+impl ToExpr for Expr {
-  fn to_expr(self) -> GenExpr { self }
+  fn to_expr(self) -> Expr { self }
 }
 
 impl<T: ToExpr> ToExpr for OrcRes<T> {
-  fn to_expr(self) -> GenExpr {
+  fn to_expr(self) -> Expr {
     match self {
-      Err(e) => botv(e),
+      Err(e) => bot(e),
       Ok(t) => t.to_expr(),
     }
   }
 }
 
 impl<A: ToAtom> ToExpr for A {
-  fn to_expr(self) -> GenExpr { atom(self) }
+  fn to_expr(self) -> Expr { atom(self) }
 }

orchid-extension/src/entrypoint.rs

@@ -6,16 +6,16 @@ use std::{mem, process, thread};
 
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::DeserAtom;
+use orchid_api::ExtMsgSet;
 use orchid_api_traits::{enc_vec, Decode, Encode};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::clone;
-use orchid_base::error::errv_to_apiv;
 use orchid_base::interner::{deintern, init_replica, sweep_replica};
 use orchid_base::logging::Logger;
+use orchid_base::macros::{mtreev_from_api, mtreev_to_api};
 use orchid_base::name::{PathSlice, Sym};
-use orchid_base::parse::Snippet;
+use orchid_base::parse::{Comment, Snippet};
-use orchid_base::reqnot::{ReqNot, Requester};
+use orchid_base::reqnot::{ReqHandlish, ReqNot, RequestHandle, Requester};
 use orchid_base::tree::{ttv_from_api, ttv_to_api};
 use substack::Substack;
 
@@ -23,12 +23,16 @@ use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo};
 use crate::atom_owned::OBJ_STORE;
 use crate::fs::VirtFS;
-use crate::lexer::{err_cascade, err_lexer_na, LexContext};
+use crate::lexer::{err_cascade, err_not_applicable, LexContext};
+use crate::macros::{apply_rule, RuleCtx};
 use crate::msg::{recv_parent_msg, send_parent_msg};
 use crate::system::{atom_by_idx, SysCtx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
 use crate::tree::{do_extra, GenTok, GenTokTree, LazyMemberFactory, TIACtxImpl};
 
+pub type ExtReq = RequestHandle<ExtMsgSet>;
+pub type ExtReqNot = ReqNot<ExtMsgSet>;
+
 pub struct ExtensionData {
   pub name: &'static str,
   pub systems: &'static [&'static dyn DynSystemCtor],
@@ -56,7 +60,7 @@ pub fn with_atom_record<T>(
   get_sys_ctx: &impl Fn(api::SysId, ReqNot<api::ExtMsgSet>) -> SysCtx,
   reqnot: ReqNot<api::ExtMsgSet>,
   atom: &api::Atom,
-  cb: impl FnOnce(&'static dyn AtomDynfo, SysCtx, api::AtomId, &[u8]) -> T,
+  cb: impl FnOnce(Box<dyn AtomDynfo>, SysCtx, api::AtomId, &[u8]) -> T,
 ) -> T {
   let mut data = &atom.data[..];
   let ctx = get_sys_ctx(atom.owner, reqnot);
@@ -107,12 +111,12 @@ fn extension_main_logic(data: ExtensionData) {
       api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) =>
         OBJ_STORE.get(atom.0).unwrap().remove().dyn_free(mk_ctx(sys_id, reqnot)),
     }),
-    clone!(systems, logger; move |req| match req.req() {
+    clone!(systems, logger; move |hand, req| match req {
-      api::HostExtReq::Ping(ping@api::Ping) => req.handle(ping, &()),
+      api::HostExtReq::Ping(ping@api::Ping) => hand.handle(&ping, &()),
-      api::HostExtReq::Sweep(sweep@api::Sweep) => req.handle(sweep, &sweep_replica()),
+      api::HostExtReq::Sweep(sweep@api::Sweep) => hand.handle(&sweep, &sweep_replica()),
       api::HostExtReq::SysReq(api::SysReq::NewSystem(new_sys)) => {
         let i = decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system).unwrap().0;
-        let cted = data.systems[i].new_system(new_sys);
+        let cted = data.systems[i].new_system(&new_sys);
         let mut vfses = HashMap::new();
         let lex_filter = cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
           let lxcf = mk_char_filter(lx.char_filter().iter().cloned());
@@ -123,7 +127,7 @@ fn extension_main_logic(data: ExtensionData) {
           cted: cted.clone(),
           id: new_sys.id,
           logger: logger.clone(),
-          reqnot: req.reqnot()
+          reqnot: hand.reqnot()
         };
         let mut tia_ctx = TIACtxImpl{
           lazy: &mut lazy_mems,
@@ -140,7 +144,7 @@ fn extension_main_logic(data: ExtensionData) {
           cted,
           lazy_members: lazy_mems
         });
-        req.handle(new_sys, &api::SystemInst {
+        hand.handle(&new_sys, &api::SystemInst {
           lex_filter,
           const_root,
           line_types: vec![]
@@ -148,16 +152,16 @@ fn extension_main_logic(data: ExtensionData) {
       }
       api::HostExtReq::GetMember(get_tree@api::GetMember(sys_id, tree_id)) => {
         let mut systems_g = systems.lock().unwrap();
-        let sys = systems_g.get_mut(sys_id).expect("System not found");
+        let sys = systems_g.get_mut(&sys_id).expect("System not found");
         let lazy = &mut sys.lazy_members;
-        let (path, cb) = match lazy.insert(*tree_id, MemberRecord::Res) {
+        let (path, cb) = match lazy.insert(tree_id, MemberRecord::Res) {
           None => panic!("Tree for ID not found"),
           Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
           Some(MemberRecord::Gen(path, cb)) => (path, cb),
         };
         let tree = cb.build(path.clone());
-        req.handle(get_tree, &tree.into_api(&mut TIACtxImpl{
+        hand.handle(&get_tree, &tree.into_api(&mut TIACtxImpl{
-            sys: SysCtx::new(*sys_id, &sys.cted, &logger, req.reqnot()),
+            sys: SysCtx::new(sys_id, &sys.cted, &logger, hand.reqnot()),
             path: Substack::Bottom,
             basepath: &path,
             lazy,
@@ -165,100 +169,124 @@ fn extension_main_logic(data: ExtensionData) {
       }
       api::HostExtReq::VfsReq(api::VfsReq::GetVfs(get_vfs@api::GetVfs(sys_id))) => {
         let systems_g = systems.lock().unwrap();
-        req.handle(get_vfs, &systems_g[sys_id].declfs)
+        hand.handle(&get_vfs, &systems_g[&sys_id].declfs)
+      }
+      api::HostExtReq::SysReq(api::SysReq::SysFwded(fwd)) => {
+        let api::SysFwded(sys_id, payload) = fwd;
+        let ctx = mk_ctx(sys_id, hand.reqnot());
+        let sys = ctx.cted.inst();
+        sys.dyn_request(hand, payload)
       }
       api::HostExtReq::VfsReq(api::VfsReq::VfsRead(vfs_read)) => {
-        let api::VfsRead(sys_id, vfs_id, path) = vfs_read;
+        let api::VfsRead(sys_id, vfs_id, path) = &vfs_read;
         let systems_g = systems.lock().unwrap();
         let path = path.iter().map(|t| deintern(*t)).collect_vec();
-        req.handle(vfs_read, &systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path)))
+        hand.handle(&vfs_read, &systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path)))
       }
-      api::HostExtReq::ParserReq(api::ParserReq::LexExpr(lex)) => {
+      api::HostExtReq::LexExpr(lex @ api::LexExpr{ sys, text, pos, id }) => {
-        let api::LexExpr{ sys, text, pos, id } = *lex;
         let systems_g = systems.lock().unwrap();
         let lexers = systems_g[&sys].cted.inst().dyn_lexers();
         mem::drop(systems_g);
         let text = deintern(text);
-        let ctx = LexContext { sys, id, pos, reqnot: req.reqnot(), text: &text };
+        let ctx = LexContext { sys, id, pos, reqnot: hand.reqnot(), text: &text };
         let trigger_char = text.chars().nth(pos as usize).unwrap();
         for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char)) {
           match lx.lex(&text[pos as usize..], &ctx) {
-            Err(e) if e.iter().any(|e| *e == err_lexer_na()) => continue,
+            Err(e) if e.any(|e| *e == err_not_applicable()) => continue,
             Err(e) => {
-              let errv = errv_to_apiv(e.iter().filter(|e| **e == err_cascade()));
+              let eopt = e.keep_only(|e| *e != err_cascade()).map(|e| Err(e.to_api()));
-              return req.handle(lex, &if errv.is_empty() { None } else { Some(Err(errv))})
+              return hand.handle(&lex, &eopt)
             },
             Ok((s, expr)) => {
-              let ctx = mk_ctx(sys, req.reqnot());
+              let ctx = mk_ctx(sys, hand.reqnot());
               let expr = expr.to_api(&mut |f, r| do_extra(f, r, ctx.clone()));
               let pos = (text.len() - s.len()) as u32;
-              return req.handle(lex, &Some(Ok(api::LexedExpr{ pos, expr })))
+              return hand.handle(&lex, &Some(Ok(api::LexedExpr{ pos, expr })))
             }
           }
         }
         writeln!(logger, "Got notified about n/a character '{trigger_char}'");
-        req.handle(lex, &None)
+        hand.handle(&lex, &None)
       },
-      api::HostExtReq::ParserReq(api::ParserReq::ParseLine(pline@api::ParseLine{ sys, line })) => {
+      api::HostExtReq::ParseLine(pline) => {
-        let mut ctx = mk_ctx(*sys, req.reqnot());
+        let api::ParseLine{ exported, comments, sys, line } = &pline;
+        let mut ctx = mk_ctx(*sys, hand.reqnot());
         let parsers = ctx.cted.inst().dyn_parsers();
+        let comments = comments.iter().map(Comment::from_api).collect();
         let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx);
         let snip = Snippet::new(line.first().expect("Empty line"), &line);
         let (head, tail) = snip.pop_front().unwrap();
         let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") };
         let parser = parsers.iter().find(|p| p.line_head() == **name).expect("No parser candidate");
-        let o_line = match parser.parse(tail) {
+        let o_line = match parser.parse(*exported, comments, tail) {
-          Err(e) => Err(errv_to_apiv(e.iter())),
+          Err(e) => Err(e.to_api()),
           Ok(t) => Ok(ttv_to_api(t, &mut |f, range| {
             api::TokenTree{ range, token: api::Token::Atom(f.clone().build(ctx.clone())) }
           })),
         };
-        req.handle(pline, &o_line)
+        hand.handle(&pline, &o_line)
       }
       api::HostExtReq::AtomReq(atom_req) => {
         let atom = atom_req.get_atom();
-        with_atom_record(&mk_ctx, req.reqnot(), atom, |nfo, ctx, id, buf| {
+        with_atom_record(&mk_ctx, hand.reqnot(), atom, |nfo, ctx, id, buf| {
           let actx = AtomCtx(buf, atom.drop, ctx.clone());
-          match atom_req {
+          match &atom_req {
             api::AtomReq::SerializeAtom(ser) => {
               let mut buf = enc_vec(&id);
               let refs = nfo.serialize(actx, &mut buf);
-              req.handle(ser, &(buf, refs))
+              hand.handle(ser, &(buf, refs))
             }
-            api::AtomReq::AtomPrint(print@api::AtomPrint(_)) => req.handle(print, &nfo.print(actx)),
+            api::AtomReq::AtomPrint(print@api::AtomPrint(_)) =>
-            api::AtomReq::AtomSame(same@api::AtomSame(_, r)) => {
+              hand.handle(print, &nfo.print(actx)),
-              // different systems or different type tags
+            api::AtomReq::Fwded(fwded) => {
-              if atom.owner != r.owner || buf != &r.data[..8] {
+              let api::Fwded(_, key, payload) = &fwded;
-                return req.handle(same, &false)
-              }
-              req.handle(same, &nfo.same(actx, r))
-            },
-            api::AtomReq::Fwded(fwded@api::Fwded(_, payload)) => {
               let mut reply = Vec::new();
-              nfo.handle_req(actx, &mut &payload[..], &mut reply);
+              let some = nfo.handle_req(actx, Sym::deintern(*key), &mut &payload[..], &mut reply);
-              req.handle(fwded, &reply)
+              hand.handle(fwded, &some.then_some(reply))
             }
-            api::AtomReq::CallRef(call@api::CallRef(_, arg))
+            api::AtomReq::CallRef(call@api::CallRef(_, arg)) => {
-              => req.handle(call, &nfo.call_ref(actx, *arg).to_api(ctx.clone())),
+              let ret = nfo.call_ref(actx, *arg);
-            api::AtomReq::FinalCall(call@api::FinalCall(_, arg))
+              hand.handle(call, &ret.api_return(ctx.clone(), &mut |h| hand.defer_drop(h)))
-              => req.handle(call, &nfo.call(actx, *arg).to_api(ctx.clone())),
+            },
-            api::AtomReq::Command(cmd@api::Command(_)) => req.handle(cmd, &match nfo.command(actx) {
+            api::AtomReq::FinalCall(call@api::FinalCall(_, arg)) => {
-              Err(e) => Err(errv_to_apiv(e.iter())),
+              let ret = nfo.call(actx, *arg);
+              hand.handle(call, &ret.api_return(ctx.clone(), &mut |h| hand.defer_drop(h)))
+            }
+            api::AtomReq::Command(cmd@api::Command(_)) => {
+              hand.handle(cmd, &match nfo.command(actx) {
+                Err(e) => Err(e.to_api()),
                 Ok(opt) => Ok(match opt {
-                Some(cont) => api::NextStep::Continue(cont.into_api(ctx.clone())),
                   None => api::NextStep::Halt,
+                  Some(cont) => api::NextStep::Continue(
+                    cont.api_return(ctx.clone(), &mut |h| hand.defer_drop(h))
+                  ),
                 })
               })
             }
+          }
         })
       },
-      api::HostExtReq::DeserAtom(deser@DeserAtom(sys, buf, refs)) => {
+      api::HostExtReq::DeserAtom(deser) => {
+        let api::DeserAtom(sys, buf, refs) = &deser;
         let mut read = &mut &buf[..];
-        let ctx = mk_ctx(*sys, req.reqnot());
+        let ctx = mk_ctx(*sys, hand.reqnot());
         let id = api::AtomId::decode(&mut read);
         let inst = ctx.cted.inst();
         let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
-        req.handle(deser, &nfo.deserialize(ctx.clone(), read, refs))
+        hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, refs))
+      },
+      orchid_api::HostExtReq::ApplyMacro(am) => {
+        let tok = hand.will_handle_as(&am);
+        let sys_ctx = mk_ctx(am.sys, hand.reqnot());
+        let ctx = RuleCtx {
+          args: am.params.into_iter().map(|(k, v)| (deintern(k), mtreev_from_api(&v))).collect(),
+          run_id: am.run_id,
+          sys: sys_ctx.clone(),
+        };
+        hand.handle_as(tok, &match apply_rule(am.id, ctx) {
+          Err(e) => e.keep_only(|e| *e != err_cascade()).map(|e| Err(e.to_api())),
+          Ok(t) => Some(Ok(mtreev_to_api(&t))),
+        })
       }
     }),
   );

orchid-extension/src/expr.rs

@@ -1,10 +1,11 @@
-use std::marker::PhantomData;
+use std::fmt;
 use std::ops::Deref;
-use std::sync::OnceLock;
+use std::sync::{Arc, OnceLock};
 
 use derive_destructure::destructure;
-use orchid_base::error::{errv_from_apiv, errv_to_apiv, OrcErr};
+use orchid_api::InspectedKind;
-use orchid_base::interner::{deintern, Tok};
+use orchid_base::error::{OrcErr, OrcErrv};
+use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::reqnot::Requester;
 
@@ -19,12 +20,13 @@ pub struct ExprHandle {
 }
 impl ExprHandle {
   pub(crate) fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
-  pub(crate) fn into_tk(self) -> api::ExprTicket {
-    let (tk, ..) = self.destructure();
-    tk
-  }
   pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() }
 }
+impl fmt::Debug for ExprHandle {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "ExprHandle({})", self.tk.0)
+  }
+}
 impl Clone for ExprHandle {
   fn clone(&self) -> Self {
     self.ctx.reqnot.notify(api::Acquire(self.ctx.id, self.tk));
@@ -35,144 +37,128 @@ impl Drop for ExprHandle {
   fn drop(&mut self) { self.ctx.reqnot.notify(api::Release(self.ctx.id, self.tk)) }
 }
 
-#[derive(Clone, destructure)]
+#[derive(Clone, Debug, destructure)]
-pub struct OwnedExpr {
+pub struct Expr {
-  pub handle: ExprHandle,
+  pub handle: Option<Arc<ExprHandle>>,
-  pub val: OnceLock<Box<GenExpr>>,
+  pub val: OnceLock<ExprData>,
 }
-impl OwnedExpr {
+impl Expr {
-  pub fn new(handle: ExprHandle) -> Self { Self { handle, val: OnceLock::new() } }
+  pub fn new(hand: Arc<ExprHandle>) -> Self { Self { handle: Some(hand), val: OnceLock::new() } }
-  pub fn get_data(&self) -> &GenExpr {
+  pub fn from_data(val: ExprData) -> Self { Self { handle: None, val: OnceLock::from(val) } }
+  pub fn get_data(&self) -> &ExprData {
     self.val.get_or_init(|| {
-      Box::new(GenExpr::from_api(
+      let handle = self.handle.as_ref().expect("Either the value or the handle must be set");
-        self.handle.ctx.reqnot.request(api::Inspect(self.handle.tk)).expr,
+      let details = handle.ctx.reqnot.request(api::Inspect { target: handle.tk });
-        &self.handle.ctx,
+      let pos = Pos::from_api(&details.location);
-      ))
+      let kind = match details.kind {
+        InspectedKind::Atom(a) => ExprKind::Atom(ForeignAtom::new(handle.clone(), a, pos.clone())),
+        InspectedKind::Bottom(b) => ExprKind::Bottom(OrcErrv::from_api(&b)),
+        InspectedKind::Opaque => ExprKind::Opaque,
+      };
+      ExprData { pos, kind }
     })
   }
   pub fn foreign_atom(self) -> Result<ForeignAtom<'static>, Self> {
-    if let GenExpr { clause: GenClause::Atom(_, atom), pos: position } = self.get_data() {
+    match (self.get_data(), &self.handle) {
-      let (atom, position) = (atom.clone(), position.clone());
+      (ExprData { kind: ExprKind::Atom(atom), .. }, Some(_)) => Ok(atom.clone()),
-      return Ok(ForeignAtom {
+      _ => Err(self),
-        ctx: self.handle.ctx.clone(),
-        expr: Some(self.handle),
-        char_marker: PhantomData,
-        pos: position,
-        atom,
-      });
-    }
-    Err(self)
     }
   }
-impl Deref for OwnedExpr {
+  pub fn api_return(
-  type Target = GenExpr;
+    self,
+    ctx: SysCtx,
+    do_slot: &mut impl FnMut(Arc<ExprHandle>),
+  ) -> api::Expression {
+    if let Some(h) = self.handle {
+      do_slot(h.clone());
+      api::Expression { location: api::Location::SlotTarget, kind: api::ExpressionKind::Slot(h.tk) }
+    } else {
+      self.val.into_inner().expect("Either value or handle must be set").api_return(ctx, do_slot)
+    }
+  }
+  pub fn handle(&self) -> Option<Arc<ExprHandle>> { self.handle.clone() }
+}
+impl Deref for Expr {
+  type Target = ExprData;
   fn deref(&self) -> &Self::Target { self.get_data() }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
-pub struct GenExpr {
+pub struct ExprData {
   pub pos: Pos,
-  pub clause: GenClause,
+  pub kind: ExprKind,
 }
-impl GenExpr {
+impl ExprData {
-  pub fn to_api(&self, ctx: SysCtx) -> api::Expr {
+  pub fn api_return(
-    api::Expr { location: self.pos.to_api(), clause: self.clause.to_api(ctx) }
+    self,
-  }
+    ctx: SysCtx,
-  pub fn into_api(self, ctx: SysCtx) -> api::Expr {
+    do_slot: &mut impl FnMut(Arc<ExprHandle>),
-    api::Expr { location: self.pos.to_api(), clause: self.clause.into_api(ctx) }
+  ) -> api::Expression {
-  }
+    api::Expression { location: self.pos.to_api(), kind: self.kind.api_return(ctx, do_slot) }
-  pub fn from_api(api: api::Expr, ctx: &SysCtx) -> Self {
-    Self { pos: Pos::from_api(&api.location), clause: GenClause::from_api(api.clause, ctx) }
   }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
-pub enum GenClause {
+pub enum ExprKind {
-  Call(Box<GenExpr>, Box<GenExpr>),
+  Call(Box<Expr>, Box<Expr>),
-  Lambda(u64, Box<GenExpr>),
+  Lambda(u64, Box<Expr>),
   Arg(u64),
-  Slot(OwnedExpr),
+  Seq(Box<Expr>, Box<Expr>),
-  Seq(Box<GenExpr>, Box<GenExpr>),
   Const(Tok<Vec<Tok<String>>>),
   NewAtom(AtomFactory),
-  Atom(api::ExprTicket, api::Atom),
+  Atom(ForeignAtom<'static>),
-  Bottom(Vec<OrcErr>),
+  Bottom(OrcErrv),
+  Opaque,
 }
-impl GenClause {
+impl ExprKind {
-  pub fn to_api(&self, ctx: SysCtx) -> api::Clause {
+  pub fn api_return(
+    self,
+    ctx: SysCtx,
+    do_slot: &mut impl FnMut(Arc<ExprHandle>),
+  ) -> api::ExpressionKind {
+    use api::ExpressionKind as K;
     match self {
       Self::Call(f, x) =>
-        api::Clause::Call(Box::new(f.to_api(ctx.clone())), Box::new(x.to_api(ctx))),
+        K::Call(Box::new(f.api_return(ctx.clone(), do_slot)), Box::new(x.api_return(ctx, do_slot))),
-      Self::Seq(a, b) => api::Clause::Seq(Box::new(a.to_api(ctx.clone())), Box::new(b.to_api(ctx))),
-      Self::Lambda(arg, body) => api::Clause::Lambda(*arg, Box::new(body.to_api(ctx))),
-      Self::Arg(arg) => api::Clause::Arg(*arg),
-      Self::Const(name) => api::Clause::Const(name.marker()),
-      Self::Bottom(err) => api::Clause::Bottom(errv_to_apiv(err)),
-      Self::NewAtom(fac) => api::Clause::NewAtom(fac.clone().build(ctx)),
-      Self::Atom(tk, atom) => api::Clause::Atom(*tk, atom.clone()),
-      Self::Slot(_) => panic!("Slot is forbidden in const tree"),
-    }
-  }
-  pub fn into_api(self, ctx: SysCtx) -> api::Clause {
-    match self {
-      Self::Call(f, x) =>
-        api::Clause::Call(Box::new(f.into_api(ctx.clone())), Box::new(x.into_api(ctx))),
       Self::Seq(a, b) =>
-        api::Clause::Seq(Box::new(a.into_api(ctx.clone())), Box::new(b.into_api(ctx))),
+        K::Seq(Box::new(a.api_return(ctx.clone(), do_slot)), Box::new(b.api_return(ctx, do_slot))),
-      Self::Lambda(arg, body) => api::Clause::Lambda(arg, Box::new(body.into_api(ctx))),
+      Self::Lambda(arg, body) => K::Lambda(arg, Box::new(body.api_return(ctx, do_slot))),
-      Self::Arg(arg) => api::Clause::Arg(arg),
+      Self::Arg(arg) => K::Arg(arg),
-      Self::Slot(extk) => api::Clause::Slot(extk.handle.into_tk()),
+      Self::Const(name) => K::Const(name.marker()),
-      Self::Const(name) => api::Clause::Const(name.marker()),
+      Self::Bottom(err) => K::Bottom(err.to_api()),
-      Self::Bottom(err) => api::Clause::Bottom(errv_to_apiv(err.iter())),
+      Self::NewAtom(fac) => K::NewAtom(fac.clone().build(ctx)),
-      Self::NewAtom(fac) => api::Clause::NewAtom(fac.clone().build(ctx)),
+      kind @ (Self::Atom(_) | Self::Opaque) => panic!("{kind:?} should have a token"),
-      Self::Atom(tk, atom) => api::Clause::Atom(tk, atom),
-    }
-  }
-  pub fn from_api(api: api::Clause, ctx: &SysCtx) -> Self {
-    match api {
-      api::Clause::Arg(id) => Self::Arg(id),
-      api::Clause::Lambda(arg, body) => Self::Lambda(arg, Box::new(GenExpr::from_api(*body, ctx))),
-      api::Clause::NewAtom(_) => panic!("Clause::NewAtom should never be received, only sent"),
-      api::Clause::Bottom(s) => Self::Bottom(errv_from_apiv(&s)),
-      api::Clause::Call(f, x) =>
-        Self::Call(Box::new(GenExpr::from_api(*f, ctx)), Box::new(GenExpr::from_api(*x, ctx))),
-      api::Clause::Seq(a, b) =>
-        Self::Seq(Box::new(GenExpr::from_api(*a, ctx)), Box::new(GenExpr::from_api(*b, ctx))),
-      api::Clause::Const(name) => Self::Const(deintern(name)),
-      api::Clause::Slot(exi) => Self::Slot(OwnedExpr::new(ExprHandle::from_args(ctx.clone(), exi))),
-      api::Clause::Atom(tk, atom) => Self::Atom(tk, atom),
     }
   }
 }
-fn inherit(clause: GenClause) -> GenExpr { GenExpr { pos: Pos::Inherit, clause } }
+fn inherit(kind: ExprKind) -> Expr { Expr::from_data(ExprData { pos: Pos::Inherit, kind }) }
 
-pub fn sym_ref(path: Tok<Vec<Tok<String>>>) -> GenExpr { inherit(GenClause::Const(path)) }
+pub fn sym_ref(path: Tok<Vec<Tok<String>>>) -> Expr { inherit(ExprKind::Const(path)) }
-pub fn atom<A: ToAtom>(atom: A) -> GenExpr { inherit(GenClause::NewAtom(atom.to_atom_factory())) }
+pub fn atom<A: ToAtom>(atom: A) -> Expr { inherit(ExprKind::NewAtom(atom.to_atom_factory())) }
 
-pub fn seq(ops: impl IntoIterator<Item = GenExpr>) -> GenExpr {
+pub fn seq(ops: impl IntoIterator<Item = Expr>) -> Expr {
-  fn recur(mut ops: impl Iterator<Item = GenExpr>) -> Option<GenExpr> {
+  fn recur(mut ops: impl Iterator<Item = Expr>) -> Option<Expr> {
     let op = ops.next()?;
     Some(match recur(ops) {
       None => op,
-      Some(rec) => inherit(GenClause::Seq(Box::new(op), Box::new(rec))),
+      Some(rec) => inherit(ExprKind::Seq(Box::new(op), Box::new(rec))),
     })
   }
   recur(ops.into_iter()).expect("Empty list provided to seq!")
 }
 
-pub fn slot(extk: OwnedExpr) -> GenClause { GenClause::Slot(extk) }
+pub fn arg(n: u64) -> ExprKind { ExprKind::Arg(n) }
 
-pub fn arg(n: u64) -> GenClause { GenClause::Arg(n) }
+pub fn lambda(n: u64, b: impl IntoIterator<Item = Expr>) -> Expr {
-
+  inherit(ExprKind::Lambda(n, Box::new(call(b))))
-pub fn lambda(n: u64, b: impl IntoIterator<Item = GenExpr>) -> GenExpr {
-  inherit(GenClause::Lambda(n, Box::new(call(b))))
 }
 
-pub fn call(v: impl IntoIterator<Item = GenExpr>) -> GenExpr {
+pub fn call(v: impl IntoIterator<Item = Expr>) -> Expr {
   v.into_iter()
-    .reduce(|f, x| inherit(GenClause::Call(Box::new(f), Box::new(x))))
+    .reduce(|f, x| inherit(ExprKind::Call(Box::new(f), Box::new(x))))
     .expect("Empty call expression")
 }
 
-pub fn bot(e: OrcErr) -> GenExpr { botv(vec![e]) }
+pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> Expr {
-pub fn botv(ev: Vec<OrcErr>) -> GenExpr { inherit(GenClause::Bottom(ev)) }
+  inherit(ExprKind::Bottom(OrcErrv::new(ev).unwrap()))
+}

orchid-extension/src/func_atom.rs

@@ -5,26 +5,25 @@ use std::sync::{Arc, Mutex};
 
 use itertools::Itertools;
 use lazy_static::lazy_static;
-use never::Never;
 use orchid_api_traits::Encode;
 use orchid_base::error::OrcRes;
 use orchid_base::interner::Tok;
 use orchid_base::name::Sym;
 use trait_set::trait_set;
 
-use crate::atom::{Atomic, ReqPck};
+use crate::atom::{MethodSet, Atomic};
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
-use crate::expr::{ExprHandle, GenExpr};
+use crate::expr::{Expr, ExprHandle};
 use crate::system::SysCtx;
 
 trait_set! {
-  trait FunCB = Fn(Vec<ExprHandle>) -> OrcRes<GenExpr> + Send + Sync + 'static;
+  trait FunCB = Fn(Vec<Expr>) -> OrcRes<Expr> + Send + Sync + 'static;
 }
 
 pub trait ExprFunc<I, O>: Clone + Send + Sync + 'static {
   const ARITY: u8;
-  fn apply(&self, v: Vec<ExprHandle>) -> OrcRes<GenExpr>;
+  fn apply(&self, v: Vec<Expr>) -> OrcRes<Expr>;
 }
 
 lazy_static! {
@@ -34,7 +33,7 @@ lazy_static! {
 #[derive(Clone)]
 pub(crate) struct Fun {
   path: Sym,
-  args: Vec<ExprHandle>,
+  args: Vec<Expr>,
   arity: u8,
   fun: Arc<dyn FunCB>,
 }
@@ -53,14 +52,14 @@ impl Fun {
 }
 impl Atomic for Fun {
   type Data = ();
-  type Req = Never;
   type Variant = OwnedVariant;
+  fn reg_reqs() -> MethodSet<Self> { MethodSet::new() }
 }
 impl OwnedAtom for Fun {
-  type Refs = Vec<ExprHandle>;
+  type Refs = Vec<Expr>;
   fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-  fn call_ref(&self, arg: ExprHandle) -> GenExpr {
+  fn call_ref(&self, arg: ExprHandle) -> Expr {
-    let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
+    let new_args = self.args.iter().cloned().chain([Expr::new(Arc::new(arg))]).collect_vec();
     if new_args.len() == self.arity.into() {
       (self.fun)(new_args).to_expr()
     } else {
@@ -68,8 +67,7 @@ impl OwnedAtom for Fun {
         .to_expr()
     }
   }
-  fn call(self, arg: ExprHandle) -> GenExpr { self.call_ref(arg) }
+  fn call(self, arg: ExprHandle) -> Expr { self.call_ref(arg) }
-  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
   fn serialize(&self, _: SysCtx, sink: &mut (impl io::Write + ?Sized)) -> Self::Refs {
     self.path.encode(sink);
     self.args.clone()
@@ -86,7 +84,7 @@ mod expr_func_derives {
 
   use super::ExprFunc;
   use crate::conv::{ToExpr, TryFromExpr};
-  use crate::func_atom::{ExprHandle, GenExpr};
+  use crate::func_atom::Expr;
 
   macro_rules! expr_func_derive {
     ($arity: tt, $($t:ident),*) => {
@@ -97,7 +95,7 @@ mod expr_func_derives {
           Func: Fn($($t,)*) -> Out + Clone + Send + Sync + 'static
         > ExprFunc<($($t,)*), Out> for Func {
           const ARITY: u8 = $arity;
-          fn apply(&self, v: Vec<ExprHandle>) -> OrcRes<GenExpr> {
+          fn apply(&self, v: Vec<Expr>) -> OrcRes<Expr> {
             assert_eq!(v.len(), Self::ARITY.into(), "Arity mismatch");
             let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above"));
             Ok(self($($t::try_from_expr([< $t:lower >])?,)*).to_expr())

orchid-extension/src/lexer.rs

@@ -5,22 +5,22 @@ use orchid_base::intern;
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::reqnot::{ReqNot, Requester};
-use orchid_base::tree::TreeHandle;
+use orchid_base::tree::TokHandle;
 
 use crate::api;
 use crate::tree::{GenTok, GenTokTree};
 
 pub fn err_cascade() -> OrcErr {
   mk_err(
-    intern!(str: "An error cascading from a recursive sublexer"),
+    intern!(str: "An error cascading from a recursive call"),
     "This error should not surface. If you are seeing it, something is wrong",
     [Pos::None.into()],
   )
 }
 
-pub fn err_lexer_na() -> OrcErr {
+pub fn err_not_applicable() -> OrcErr {
   mk_err(
-    intern!(str: "Pseudo-error to communicate that the lexer doesn't apply"),
+    intern!(str: "Pseudo-error to communicate that the current branch in a dispatch doesn't apply"),
     &*err_cascade().message,
     [Pos::None.into()],
   )
@@ -38,7 +38,7 @@ impl<'a> LexContext<'a> {
     let start = self.pos(tail);
     let lx =
       self.reqnot.request(api::SubLex { pos: start, id: self.id }).ok_or_else(err_cascade)?;
-    Ok((&self.text[lx.pos as usize..], GenTok::Slot(TreeHandle::new(lx.ticket)).at(start..lx.pos)))
+    Ok((&self.text[lx.pos as usize..], GenTok::Slot(TokHandle::new(lx.ticket)).at(start..lx.pos)))
   }
 
   pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }

orchid-extension/src/lib.rs

@@ -16,3 +16,5 @@ pub mod parser;
 pub mod system;
 pub mod system_ctor;
 pub mod tree;
+pub mod macros;
+pub mod api_conv;

orchid-extension/src/macros.rs

@@ -0,0 +1,89 @@
+use ahash::HashMap;
+use lazy_static::lazy_static;
+use orchid_base::{error::OrcRes, interner::{intern, Tok}, location::Pos, macros::{mtreev_from_api, mtreev_to_api, MTree}, parse::Comment, reqnot::Requester};
+use trait_set::trait_set;
+use crate::{api, lexer::err_cascade, system::SysCtx};
+use std::{num::NonZero, sync::RwLock};
+
+pub trait Macro {
+  fn pattern() -> MTree<'static>;
+  fn apply(binds: HashMap<Tok<String>, MTree<'_>>) -> MTree<'_>;
+}
+
+pub trait DynMacro {
+  fn pattern(&self) -> MTree<'static>;
+  fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a>>) -> MTree<'a>;
+}
+
+impl<T: Macro> DynMacro for T {
+  fn pattern(&self) -> MTree<'static> { Self::pattern() }
+  fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a>>) -> MTree<'a> { Self::apply(binds) }
+}
+
+pub struct RuleCtx<'a> {
+  pub(crate) args: HashMap<Tok<String>, Vec<MTree<'a>>>,
+  pub(crate) run_id: api::ParsId,
+  pub(crate) sys: SysCtx,
+}
+impl<'a> RuleCtx<'a> {
+  pub fn recurse(&mut self, tree: &[MTree<'a>]) -> OrcRes<Vec<MTree<'a>>> {
+    let req = api::RunMacros{ run_id: self.run_id, query: mtreev_to_api(tree) };
+    Ok(mtreev_from_api(&self.sys.reqnot.request(req).ok_or_else(err_cascade)?))
+  }
+  pub fn getv(&mut self, key: &Tok<String>) -> Vec<MTree<'a>> {
+    self.args.remove(key).expect("Key not found")
+  }
+  pub fn gets(&mut self, key: &Tok<String>) -> MTree<'a> {
+    let v = self.getv(key);
+    assert!(v.len() == 1, "Not a scalar");
+    v.into_iter().next().unwrap()
+  }
+  pub fn unused_arg<'b>(&mut self, keys: impl IntoIterator<Item = &'b Tok<String>>) {
+    keys.into_iter().for_each(|k| {self.getv(k);});
+  }
+}
+
+trait_set! {
+  pub trait RuleCB = for<'a> Fn(RuleCtx<'a>) -> OrcRes<Vec<MTree<'a>>> + Send + Sync;
+}
+
+lazy_static!{
+  static ref RULES: RwLock<HashMap<api::MacroId, Box<dyn RuleCB>>> = RwLock::default();
+}
+
+pub struct Rule {
+  pub(crate) comments: Vec<Comment>,
+  pub(crate) pattern: Vec<MTree<'static>>,
+  pub(crate) id: api::MacroId,
+}
+impl Rule {
+  pub(crate) fn to_api(&self) -> api::MacroRule {
+    api::MacroRule {
+      comments: self.comments.iter().map(|c| c.to_api()).collect(),
+      location: api::Location::Inherit,
+      pattern: mtreev_to_api(&self.pattern),
+      id: self.id,
+    }
+  }
+}
+
+pub fn rule_cmt<'a>(
+  cmt: impl IntoIterator<Item = &'a str>,
+  pattern: Vec<MTree<'static>>,
+  apply: impl RuleCB + 'static
+) -> Rule {
+  let mut rules = RULES.write().unwrap();
+  let id = api::MacroId(NonZero::new(rules.len() as u64 + 1).unwrap());
+  rules.insert(id, Box::new(apply));
+  let comments = cmt.into_iter().map(|s| Comment { pos: Pos::Inherit, text: intern(s) }).collect();
+  Rule { comments, pattern, id }
+}
+
+pub fn rule(pattern: Vec<MTree<'static>>, apply: impl RuleCB + 'static) -> Rule {
+  rule_cmt([], pattern, apply)
+}
+
+pub(crate) fn apply_rule(id: api::MacroId, ctx: RuleCtx<'static>) -> OrcRes<Vec<MTree<'static>>> {
+  let rules = RULES.read().unwrap();
+  rules[&id](ctx)
+}

orchid-extension/src/parser.rs

@@ -1,5 +1,5 @@
 use orchid_base::error::OrcRes;
-use orchid_base::parse::Snippet;
+use orchid_base::parse::{Comment, Snippet};
 
 use crate::atom::{AtomFactory, ForeignAtom};
 use crate::tree::GenTokTree;
@@ -8,17 +8,33 @@ pub type GenSnippet<'a> = Snippet<'a, 'a, ForeignAtom<'a>, AtomFactory>;
 
 pub trait Parser: Send + Sync + Sized + Default + 'static {
   const LINE_HEAD: &'static str;
-  fn parse(line: GenSnippet<'_>) -> OrcRes<Vec<GenTokTree<'_>>>;
+  fn parse(
+    exported: bool,
+    comments: Vec<Comment>,
+    line: GenSnippet<'_>,
+  ) -> OrcRes<Vec<GenTokTree<'_>>>;
 }
 
 pub trait DynParser: Send + Sync + 'static {
   fn line_head(&self) -> &'static str;
-  fn parse<'a>(&self, line: GenSnippet<'a>) -> OrcRes<Vec<GenTokTree<'a>>>;
+  fn parse<'a>(
+    &self,
+    exported: bool,
+    comments: Vec<Comment>,
+    line: GenSnippet<'a>,
+  ) -> OrcRes<Vec<GenTokTree<'a>>>;
 }
 
 impl<T: Parser> DynParser for T {
   fn line_head(&self) -> &'static str { Self::LINE_HEAD }
-  fn parse<'a>(&self, line: GenSnippet<'a>) -> OrcRes<Vec<GenTokTree<'a>>> { Self::parse(line) }
+  fn parse<'a>(
+    &self,
+    exported: bool,
+    comments: Vec<Comment>,
+    line: GenSnippet<'a>,
+  ) -> OrcRes<Vec<GenTokTree<'a>>> {
+    Self::parse(exported, comments, line)
+  }
 }
 
 pub type ParserObj = &'static dyn DynParser;

orchid-extension/src/system.rs

@@ -3,96 +3,103 @@ use std::num::NonZero;
 use std::sync::Arc;
 
 use hashbrown::HashMap;
-use orchid_api::AtomId;
+use orchid_api_traits::{Coding, Decode};
-use orchid_api_traits::Decode;
+use orchid_base::boxed_iter::BoxedIter;
 use orchid_base::interner::Tok;
 use orchid_base::logging::Logger;
-use orchid_base::reqnot::ReqNot;
+use orchid_base::reqnot::{Receipt, ReqNot};
 
 use crate::api;
 use crate::atom::{get_info, AtomCtx, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom};
+use crate::entrypoint::ExtReq;
 use crate::fs::DeclFs;
 // use crate::fun::Fun;
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
 use crate::system_ctor::{CtedObj, SystemCtor};
-use crate::tree::GenMemberKind;
+use crate::tree::MemKind;
 
 /// System as consumed by foreign code
 pub trait SystemCard: Default + Send + Sync + 'static {
   type Ctor: SystemCtor;
-  const ATOM_DEFS: &'static [Option<&'static dyn AtomDynfo>];
+  type Req: Coding;
+  fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>>;
 }
 
 pub trait DynSystemCard: Send + Sync + 'static {
   fn name(&self) -> &'static str;
   /// Atoms explicitly defined by the system card. Do not rely on this for
   /// querying atoms as it doesn't include the general atom types
-  fn atoms(&self) -> &'static [Option<&'static dyn AtomDynfo>];
+  fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>>;
 }
 
 /// Atoms supported by this package which may appear in all extensions.
 /// The indices of these are bitwise negated, such that the MSB of an atom index
 /// marks whether it belongs to this package (0) or the importer (1)
-fn general_atoms() -> &'static [Option<&'static dyn AtomDynfo>] {
+fn general_atoms() -> impl Iterator<Item = Option<Box<dyn AtomDynfo>>> {
-  &[/*Some(Fun::INFO)*/]
+  [/*Some(Fun::INFO)*/].into_iter()
 }
 
 pub fn atom_info_for(
   sys: &(impl DynSystemCard + ?Sized),
   tid: TypeId,
-) -> Option<(api::AtomId, &'static dyn AtomDynfo)> {
+) -> Option<(api::AtomId, Box<dyn AtomDynfo>)> {
-  (sys.atoms().iter().enumerate().map(|(i, o)| (NonZero::new(i as u64 + 1).unwrap(), o)))
+  (sys.atoms().enumerate().map(|(i, o)| (NonZero::new(i as u64 + 1).unwrap(), o)))
-    .chain(general_atoms().iter().enumerate().map(|(i, o)| (NonZero::new(!(i as u64)).unwrap(), o)))
+    .chain(general_atoms().enumerate().map(|(i, o)| (NonZero::new(!(i as u64)).unwrap(), o)))
-    .filter_map(|(i, o)| o.as_ref().map(|a| (api::AtomId(i), *a)))
+    .filter_map(|(i, o)| o.map(|a| (api::AtomId(i), a)))
     .find(|ent| ent.1.tid() == tid)
 }
 
 pub fn atom_by_idx(
   sys: &(impl DynSystemCard + ?Sized),
   tid: api::AtomId,
-) -> Option<&'static dyn AtomDynfo> {
+) -> Option<Box<dyn AtomDynfo>> {
   if (u64::from(tid.0) >> (u64::BITS - 1)) & 1 == 1 {
-    general_atoms()[!u64::from(tid.0) as usize]
+    general_atoms().nth(!u64::from(tid.0) as usize).unwrap()
   } else {
-    sys.atoms()[u64::from(tid.0) as usize - 1]
+    sys.atoms().nth(u64::from(tid.0) as usize - 1).unwrap()
   }
 }
 
 pub fn resolv_atom(
   sys: &(impl DynSystemCard + ?Sized),
   atom: &api::Atom,
-) -> &'static dyn AtomDynfo {
+) -> Box<dyn AtomDynfo> {
   let tid = api::AtomId::decode(&mut &atom.data[..8]);
   atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
 impl<T: SystemCard> DynSystemCard for T {
   fn name(&self) -> &'static str { T::Ctor::NAME }
-  fn atoms(&self) -> &'static [Option<&'static dyn AtomDynfo>] { Self::ATOM_DEFS }
+  fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>> { Box::new(Self::atoms().into_iter()) }
 }
 
 /// System as defined by author
 pub trait System: Send + Sync + SystemCard + 'static {
-  fn env() -> Vec<(Tok<String>, GenMemberKind)>;
+  fn env() -> Vec<(Tok<String>, MemKind)>;
   fn vfs() -> DeclFs;
   fn lexers() -> Vec<LexerObj>;
   fn parsers() -> Vec<ParserObj>;
+  fn request(hand: ExtReq, req: Self::Req) -> Receipt;
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
-  fn dyn_env(&self) -> HashMap<Tok<String>, GenMemberKind>;
+  fn dyn_env(&self) -> HashMap<Tok<String>, MemKind>;
   fn dyn_vfs(&self) -> DeclFs;
   fn dyn_lexers(&self) -> Vec<LexerObj>;
   fn dyn_parsers(&self) -> Vec<ParserObj>;
+  fn dyn_request(&self, hand: ExtReq, req: Vec<u8>) -> Receipt;
   fn card(&self) -> &dyn DynSystemCard;
 }
 
 impl<T: System> DynSystem for T {
-  fn dyn_env(&self) -> HashMap<Tok<String>, GenMemberKind> { Self::env().into_iter().collect() }
+  fn dyn_env(&self) -> HashMap<Tok<String>, MemKind> { Self::env().into_iter().collect() }
   fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
   fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
   fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
+  fn dyn_request(&self, hand: ExtReq, req: Vec<u8>) -> Receipt {
+    Self::request(hand, <Self as SystemCard>::Req::decode(&mut &req[..]))
+  }
   fn card(&self) -> &dyn DynSystemCard { self }
 }
 
@@ -101,7 +108,7 @@ pub fn downcast_atom<A: AtomicFeatures>(foreign: ForeignAtom) -> Result<TypAtom<
   let ctx = foreign.ctx.clone();
   let info_ent = (ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
     .map(|sys| get_info::<A>(sys.get_card()))
-    .filter(|(pos, _)| AtomId::decode(&mut data) == *pos);
+    .filter(|(pos, _)| api::AtomId::decode(&mut data) == *pos);
   match info_ent {
     None => Err(foreign),
     Some((_, info)) => {

orchid-extension/src/tree.rs

@@ -1,6 +1,5 @@
 use std::num::NonZero;
 use std::ops::Range;
-use std::sync::Arc;
 
 use dyn_clone::{clone_box, DynClone};
 use hashbrown::HashMap;
@@ -8,7 +7,9 @@ use itertools::Itertools;
 use orchid_base::interner::{intern, Tok};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
+use orchid_base::parse::Comment;
 use orchid_base::tree::{TokTree, Token};
+use ordered_float::NotNan;
 use substack::Substack;
 use trait_set::trait_set;
 
@@ -16,8 +17,9 @@ use crate::api;
 use crate::atom::{AtomFactory, ForeignAtom};
 use crate::conv::ToExpr;
 use crate::entrypoint::MemberRecord;
-use crate::expr::GenExpr;
+use crate::expr::Expr;
 use crate::func_atom::{ExprFunc, Fun};
+use crate::macros::Rule;
 use crate::system::SysCtx;
 
 pub type GenTokTree<'a> = TokTree<'a, ForeignAtom<'a>, AtomFactory>;
@@ -27,68 +29,85 @@ pub fn do_extra(f: &AtomFactory, r: Range<u32>, ctx: SysCtx) -> api::TokenTree {
   api::TokenTree { range: r, token: api::Token::Atom(f.clone().build(ctx)) }
 }
 
+fn with_export(mem: GenMember, public: bool) -> Vec<GenItem> {
+  (public.then(|| GenItemKind::Export(mem.name.clone()).at(Pos::Inherit)).into_iter())
+    .chain([GenItemKind::Member(mem).at(Pos::Inherit)])
+    .collect()
+}
+
 pub struct GenItem {
-  pub item: GenItemKind,
+  pub kind: GenItemKind,
-  pub comments: Vec<(String, Pos)>,
+  pub comments: Vec<Comment>,
   pub pos: Pos,
 }
 impl GenItem {
   pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Item {
-    let kind = match self.item {
+    let kind = match self.kind {
-      GenItemKind::Raw(item) => api::ItemKind::Raw(Vec::from_iter(
+      GenItemKind::Export(n) => api::ItemKind::Export(n.marker()),
-        item.into_iter().map(|t| t.to_api(&mut |f, r| do_extra(f, r, ctx.sys()))),
-      )),
       GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx)),
+      GenItemKind::Import(cn) => api::ItemKind::Import(cn.tok().marker()),
+      GenItemKind::Macro(prio, rules) => api::ItemKind::Macro(api::MacroBlock {
+        priority: prio,
+        rules: rules.into_iter().map(|r| r.to_api() ).collect_vec(),
+      })
     };
-    let comments = self.comments.into_iter().map(|(s, p)| (Arc::new(s), p.to_api())).collect_vec();
+    let comments = self.comments.into_iter().map(|c| c.to_api()).collect_vec();
     api::Item { location: self.pos.to_api(), comments, kind }
   }
 }
 
-pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> GenItem {
+pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenItem> {
-  let kind = GenMemberKind::Const(value.to_expr());
+  with_export(GenMember { name: intern(name), kind: MemKind::Const(value.to_expr()) }, public)
-  GenItemKind::Member(GenMember { exported: public, name: intern(name), kind }).at(Pos::Inherit)
 }
 pub fn module(
   public: bool,
   name: &str,
   imports: impl IntoIterator<Item = Sym>,
-  items: impl IntoIterator<Item = GenItem>,
+  items: impl IntoIterator<Item = Vec<GenItem>>,
-) -> GenItem {
+) -> Vec<GenItem> {
   let (name, kind) = root_mod(name, imports, items);
-  GenItemKind::Member(GenMember { exported: public, name, kind }).at(Pos::Inherit)
+  with_export(GenMember { name, kind }, public)
 }
 pub fn root_mod(
   name: &str,
   imports: impl IntoIterator<Item = Sym>,
-  items: impl IntoIterator<Item = GenItem>,
+  items: impl IntoIterator<Item = Vec<GenItem>>,
-) -> (Tok<String>, GenMemberKind) {
+) -> (Tok<String>, MemKind) {
-  let kind = GenMemberKind::Mod {
+  let kind = MemKind::Mod {
     imports: imports.into_iter().collect(),
-    items: items.into_iter().collect(),
+    items: items.into_iter().flatten().collect(),
   };
   (intern(name), kind)
 }
-pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> GenItem {
+pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenItem> {
-  let fac = LazyMemberFactory::new(move |sym| GenMemberKind::Const(Fun::new(sym, xf).to_expr()));
+  let fac = LazyMemberFactory::new(move |sym| MemKind::Const(Fun::new(sym, xf).to_expr()));
-  let mem = GenMember { exported, name: intern(name), kind: GenMemberKind::Lazy(fac) };
+  with_export(GenMember { name: intern(name), kind: MemKind::Lazy(fac) }, exported)
-  GenItemKind::Member(mem).at(Pos::Inherit)
+}
+pub fn macro_block(prio: Option<f64>, rules: impl IntoIterator<Item = Rule>) -> Vec<GenItem> {
+  let prio = prio.map(|p| NotNan::new(p).unwrap());
+  vec![GenItemKind::Macro(prio, rules.into_iter().collect_vec()).gen()]
 }
 
-pub fn comments<'a>(cmts: impl IntoIterator<Item = &'a str>, mut val: GenItem) -> GenItem {
+pub fn comments<'a>(
-  val.comments.extend(cmts.into_iter().map(|c| (c.to_string(), Pos::Inherit)));
+  cmts: impl IntoIterator<Item = &'a str> + Clone,
+  mut val: Vec<GenItem>,
+) -> Vec<GenItem> {
+  for v in val.iter_mut() {
+    v.comments
+      .extend(cmts.clone().into_iter().map(|c| Comment { text: intern(c), pos: Pos::Inherit }));
+  }
   val
 }
 
 trait_set! {
-  trait LazyMemberCallback = FnOnce(Sym) -> GenMemberKind + Send + Sync + DynClone
+  trait LazyMemberCallback = FnOnce(Sym) -> MemKind + Send + Sync + DynClone
 }
 pub struct LazyMemberFactory(Box<dyn LazyMemberCallback>);
 impl LazyMemberFactory {
-  pub fn new(cb: impl FnOnce(Sym) -> GenMemberKind + Send + Sync + Clone + 'static) -> Self {
+  pub fn new(cb: impl FnOnce(Sym) -> MemKind + Send + Sync + Clone + 'static) -> Self {
     Self(Box::new(cb))
   }
-  pub fn build(self, path: Sym) -> GenMemberKind { (self.0)(path) }
+  pub fn build(self, path: Sym) -> MemKind { (self.0)(path) }
 }
 impl Clone for LazyMemberFactory {
   fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
@@ -96,42 +115,48 @@ impl Clone for LazyMemberFactory {
 
 pub enum GenItemKind {
   Member(GenMember),
-  Raw(Vec<GenTokTree<'static>>),
+  Export(Tok<String>),
+  Import(Sym),
+  Macro(Option<NotNan<f64>>, Vec<Rule>),
 }
 impl GenItemKind {
-  pub fn at(self, position: Pos) -> GenItem {
+  pub fn at(self, pos: Pos) -> GenItem { GenItem { kind: self, comments: vec![], pos } }
-    GenItem { item: self, comments: vec![], pos: position }
+  pub fn gen(self) -> GenItem { GenItem { kind: self, comments: vec![], pos: Pos::Inherit } }
+  pub fn gen_equiv(self, comments: Vec<Comment>) -> GenItem {
+    GenItem { kind: self, comments, pos: Pos::Inherit }
   }
 }
 
 pub struct GenMember {
-  exported: bool,
   name: Tok<String>,
-  kind: GenMemberKind,
+  kind: MemKind,
 }
 impl GenMember {
   pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member {
     api::Member {
       name: self.name.marker(),
-      exported: self.exported,
       kind: self.kind.into_api(&mut ctx.push_path(self.name)),
     }
   }
 }
 
-pub enum GenMemberKind {
+pub enum MemKind {
-  Const(GenExpr),
+  Const(Expr),
   Mod { imports: Vec<Sym>, items: Vec<GenItem> },
   Lazy(LazyMemberFactory),
 }
-impl GenMemberKind {
+impl MemKind {
   pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
     match self {
       Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
-      Self::Const(c) => api::MemberKind::Const(c.into_api(ctx.sys())),
+      Self::Const(c) =>
+        api::MemberKind::Const(c.api_return(ctx.sys(), &mut |_| panic!("Slot found in const tree"))),
       Self::Mod { imports, items } => api::MemberKind::Module(api::Module {
-        imports: imports.into_iter().map(|t| t.tok().marker()).collect(),
+        items: (imports.into_iter())
-        items: items.into_iter().map(|i| i.into_api(ctx)).collect_vec(),
+          .map(|t| GenItemKind::Import(t).gen())
+          .chain(items)
+          .map(|i| i.into_api(ctx))
+          .collect_vec(),
       }),
     }
   }

orchid-host/Cargo.toml

@@ -11,9 +11,11 @@ hashbrown = "0.14.5"
 itertools = "0.13.0"
 lazy_static = "1.4.0"
 never = "0.1.0"
+num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "4.2.0"
 paste = "1.0.15"
-substack = "1.1.0"
+substack = "1.1.1"
+trait-set = "0.3.0"

orchid-host/src/expr.rs

@@ -1,24 +1,34 @@
+use std::collections::VecDeque;
 use std::num::NonZeroU64;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 
 use hashbrown::HashMap;
 use lazy_static::lazy_static;
+use orchid_base::error::OrcErrv;
+use orchid_base::interner::deintern;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::tree::AtomTok;
 
 use crate::api;
-use crate::extension::{AtomHand, System};
+use crate::extension::AtomHand;
+
+pub type ExprParseCtx = ();
 
 #[derive(Clone, Debug)]
-pub struct RtExpr {
+pub struct Expr {
   is_canonical: Arc<AtomicBool>,
-  data: Arc<()>,
+  pos: Pos,
+  kind: Arc<RwLock<ExprKind>>,
 }
-impl RtExpr {
+impl Expr {
+  pub fn pos(&self) -> Pos { self.pos.clone() }
   pub fn as_atom(&self) -> Option<AtomHand> { todo!() }
   pub fn strong_count(&self) -> usize { todo!() }
   pub fn id(&self) -> api::ExprTicket {
     api::ExprTicket(
-      NonZeroU64::new(self.data.as_ref() as *const () as usize as u64)
+      NonZeroU64::new(self.kind.as_ref() as *const RwLock<_> as usize as u64)
         .expect("this is a ref, it cannot be null"),
     )
   }
@@ -31,14 +41,29 @@ impl RtExpr {
   pub fn resolve(tk: api::ExprTicket) -> Option<Self> {
     KNOWN_EXPRS.read().unwrap().get(&tk).cloned()
   }
-  pub fn from_api(api: api::Expr, sys: &System) -> Self {
+  pub fn from_api(api: api::Expression, ctx: &mut ExprParseCtx) -> Self {
-    Self { data: Arc::default(), is_canonical: Arc::default() }
+    if let api::ExpressionKind::Slot(tk) = &api.kind {
+      return Self::resolve(*tk).expect("Invalid slot");
+    }
+    Self {
+      kind: Arc::new(RwLock::new(ExprKind::from_api(api.kind, ctx))),
+      is_canonical: Arc::default(),
+      pos: Pos::from_api(&api.location),
     }
   }
-impl Drop for RtExpr {
+  pub fn to_api(&self) -> api::InspectedKind {
+    use api::InspectedKind as K;
+    match &*self.kind.read().unwrap() {
+      ExprKind::Atom(a) => K::Atom(a.to_api()),
+      ExprKind::Bottom(b) => K::Bottom(b.to_api()),
+      _ => K::Opaque,
+    }
+  }
+}
+impl Drop for Expr {
   fn drop(&mut self) {
     // If the only two references left are this and known, remove from known
-    if Arc::strong_count(&self.data) == 2 && self.is_canonical.load(Ordering::Relaxed) {
+    if Arc::strong_count(&self.kind) == 2 && self.is_canonical.load(Ordering::Relaxed) {
       // if known is poisoned, a leak is preferable to a panicking destructor
       if let Ok(mut w) = KNOWN_EXPRS.write() {
         w.remove(&self.id());
@@ -48,5 +73,67 @@ impl Drop for RtExpr {
 }
 
 lazy_static! {
-  static ref KNOWN_EXPRS: RwLock<HashMap<api::ExprTicket, RtExpr>> = RwLock::default();
+  static ref KNOWN_EXPRS: RwLock<HashMap<api::ExprTicket, Expr>> = RwLock::default();
+}
+
+#[derive(Clone, Debug)]
+pub enum ExprKind {
+  Seq(Expr, Expr),
+  Call(Expr, Expr),
+  Atom(AtomHand),
+  Argument,
+  Lambda(Option<PathSet>, Expr),
+  Bottom(OrcErrv),
+  Const(Sym),
+}
+impl ExprKind {
+  pub fn from_api(api: api::ExpressionKind, ctx: &mut ExprParseCtx) -> Self {
+    use api::ExpressionKind as K;
+    match api {
+      K::Slot(_) => panic!("Handled in Expr"),
+      K::Lambda(id, b) => ExprKind::Lambda(PathSet::from_api(id, &b), Expr::from_api(*b, ctx)),
+      K::Arg(_) => ExprKind::Argument,
+      K::Bottom(b) => ExprKind::Bottom(OrcErrv::from_api(&b)),
+      K::Call(f, x) => ExprKind::Call(Expr::from_api(*f, ctx), Expr::from_api(*x, ctx)),
+      K::Const(c) => ExprKind::Const(Sym::from_tok(deintern(c)).unwrap()),
+      K::NewAtom(a) => ExprKind::Atom(AtomHand::from_api(a)),
+      K::Seq(a, b) => ExprKind::Seq(Expr::from_api(*a, ctx), Expr::from_api(*b, ctx)),
+    }
+  }
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub enum Step {
+  Left,
+  Right,
+}
+
+#[derive(Clone, Debug)]
+pub struct PathSet {
+  /// The single steps through [super::nort::Clause::Apply]
+  pub steps: VecDeque<Step>,
+  /// if Some, it splits at a [super::nort::Clause::Apply]. If None, it ends in
+  /// a [super::nort::Clause::LambdaArg]
+  pub next: Option<(Box<PathSet>, Box<PathSet>)>,
+}
+impl PathSet {
+  pub fn after(mut self, step: Step) -> Self {
+    self.steps.push_front(step);
+    self
+  }
+  pub fn from_api(id: u64, b: &api::Expression) -> Option<Self> {
+    use api::ExpressionKind as K;
+    match &b.kind {
+      K::Arg(id2) => (id == *id2).then(|| Self { steps: VecDeque::new(), next: None }),
+      K::Bottom(_) | K::Const(_) | K::NewAtom(_) | K::Slot(_) => None,
+      K::Lambda(_, b) => Self::from_api(id, b),
+      K::Call(l, r) | K::Seq(l, r) => match (Self::from_api(id, l), Self::from_api(id, r)) {
+        (Some(a), Some(b)) =>
+          Some(Self { steps: VecDeque::new(), next: Some((Box::new(a), Box::new(b))) }),
+        (Some(l), None) => Some(l.after(Step::Left)),
+        (None, Some(r)) => Some(r.after(Step::Right)),
+        (None, None) => None,
+      },
+    }
+  }
 }

orchid-host/src/extension.rs

@@ -11,19 +11,23 @@ use hashbrown::hash_map::Entry;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use lazy_static::lazy_static;
-use orchid_api_traits::{enc_vec, Decode, Request};
+use orchid_api::TStrv;
+use orchid_api_traits::Request;
 use orchid_base::char_filter::char_filter_match;
-use orchid_base::error::{errv_from_apiv, mk_err, OrcRes};
+use orchid_base::error::{OrcErrv, OrcRes};
 use orchid_base::interner::{deintern, intern, Tok};
 use orchid_base::logging::Logger;
+use orchid_base::macros::{mtreev_from_api, mtreev_to_api};
+use orchid_base::parse::Comment;
 use orchid_base::reqnot::{ReqNot, Requester as _};
-use orchid_base::tree::{ttv_from_api, AtomInTok};
+use orchid_base::tree::{ttv_from_api, AtomTok};
-use orchid_base::{clone, intern};
+use orchid_base::clone;
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
 use crate::api;
-use crate::expr::RtExpr;
+use crate::expr::Expr;
+use crate::macros::macro_recur;
 use crate::tree::{Member, ParsTokTree};
 
 #[derive(Debug, destructure)]
@@ -76,7 +80,7 @@ impl AtomHand {
       Self::create_new(atom)
     }
   }
-  pub fn call(self, arg: RtExpr) -> api::Expr {
+  pub fn call(self, arg: Expr) -> api::Expression {
     let owner_sys = self.0.owner.clone();
     let reqnot = owner_sys.reqnot();
     let ticket = owner_sys.give_expr(arg.canonicalize(), || arg);
@@ -85,20 +89,13 @@ impl AtomHand {
       Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), ticket)),
     }
   }
-  pub fn same(&self, other: &AtomHand) -> bool {
+  pub fn req(&self, key: TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
-    let owner = self.0.owner.id();
+    self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req))
-    if other.0.owner.id() != owner {
-      return false;
-    }
-    self.0.owner.reqnot().request(api::AtomSame(self.0.api_ref(), other.0.api_ref()))
-  }
-  pub fn req(&self, req: Vec<u8>) -> Vec<u8> {
-    self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), req))
   }
   pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
   pub fn print(&self) -> String { self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())) }
 }
-impl AtomInTok for AtomHand {
+impl AtomTok for AtomHand {
   type Context = ();
   fn from_api(atom: &orchid_api::Atom, _: Range<u32>, (): &mut Self::Context) -> Self {
     Self::from_api(atom.clone())
@@ -118,6 +115,7 @@ impl fmt::Display for AtomHand {
 pub trait ExtensionPort: Send + Sync {
   fn send(&self, msg: &[u8]);
   fn receive(&self) -> Option<Vec<u8>>;
+  fn header(&self) -> &api::ExtensionHeader;
 }
 
 /// Data held about an Extension. This is refcounted within [Extension]. It's
@@ -139,7 +137,7 @@ impl Drop for ExtensionData {
 
 fn acq_expr(sys: api::SysId, extk: api::ExprTicket) {
   (System::resolve(sys).expect("Expr acq'd by invalid system"))
-    .give_expr(extk, || RtExpr::resolve(extk).expect("Invalid expr acq'd"));
+    .give_expr(extk, || Expr::resolve(extk).expect("Invalid expr acq'd"));
 }
 
 fn rel_expr(sys: api::SysId, extk: api::ExprTicket) {
@@ -154,10 +152,11 @@ fn rel_expr(sys: api::SysId, extk: api::ExprTicket) {
 pub struct Extension(Arc<ExtensionData>);
 impl Extension {
   pub fn new_process(port: Arc<dyn ExtensionPort>, logger: Logger) -> io::Result<Self> {
-    port.send(&enc_vec(&api::HostHeader { log_strategy: logger.strat() }));
+    let eh = port.header();
-    let header_reply = port.receive().expect("Extension exited immediately");
-    let eh = api::ExtensionHeader::decode(&mut &header_reply[..]);
     let ret = Arc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
+      systems: (eh.systems.iter().cloned())
+        .map(|decl| SystemCtor { decl, ext: weak.clone() })
+        .collect(),
       logger,
       port: port.clone(),
       reqnot: ReqNot::new(
@@ -175,46 +174,43 @@ impl Extension {
           },
           api::ExtHostNotif::Log(api::Log(str)) => weak.upgrade().unwrap().logger.log(str),
         }),
-        |req| match req.req() {
+        |hand, req| match req {
-          api::ExtHostReq::Ping(ping) => req.handle(ping, &()),
+          api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()),
           api::ExtHostReq::IntReq(intreq) => match intreq {
-            api::IntReq::InternStr(s) => req.handle(s, &intern(&**s.0).marker()),
+            api::IntReq::InternStr(s) => hand.handle(&s, &intern(&**s.0).marker()),
-            api::IntReq::InternStrv(v) => req.handle(v, &intern(&*v.0).marker()),
+            api::IntReq::InternStrv(v) => hand.handle(&v, &intern(&*v.0).marker()),
-            api::IntReq::ExternStr(si) => req.handle(si, &deintern(si.0).arc()),
+            api::IntReq::ExternStr(si) => hand.handle(&si, &deintern(si.0).arc()),
             api::IntReq::ExternStrv(vi) =>
-              req.handle(vi, &Arc::new(deintern(vi.0).iter().map(|t| t.marker()).collect_vec())),
+              hand.handle(&vi, &Arc::new(deintern(vi.0).iter().map(|t| t.marker()).collect_vec())),
           },
-          api::ExtHostReq::Fwd(fw @ api::Fwd(atom, _body)) => {
+          api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
             let sys = System::resolve(atom.owner).unwrap();
-            req.handle(fw, &sys.reqnot().request(api::Fwded(fw.0.clone(), fw.1.clone())))
+            hand.handle(fw, &sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())))
+          },
+          api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
+            let sys = System::resolve(id).unwrap();
+            hand.handle(fw, &sys.request(body.clone()))
           },
           api::ExtHostReq::SubLex(sl) => {
             let (rep_in, rep_out) = sync_channel(0);
             let lex_g = LEX_RECUR.lock().unwrap();
             let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
             req_in.send(ReqPair(sl.clone(), rep_in)).unwrap();
-            req.handle(sl, &rep_out.recv().unwrap())
+            hand.handle(&sl, &rep_out.recv().unwrap())
-          },
-          api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect(tk))) => {
-            let expr = RtExpr::resolve(*tk);
-            req.handle(ins, &api::Details {
-              refcount: 1,
-              expr: api::Expr {
-                location: api::Location::None,
-                clause: api::Clause::Bottom(vec![
-                  mk_err(
-                    intern!(str: "Unsupported"),
-                    "Inspecting clauses is unsupported at the moment",
-                    [],
-                  )
-                  .to_api(),
-                ]),
           },
+          api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => {
+            let expr = Expr::resolve(target).expect("Invalid ticket");
+            hand.handle(&ins, &api::Inspected {
+              refcount: expr.strong_count() as u32,
+              location: expr.pos().to_api(),
+              kind: expr.to_api(),
             })
           },
+          api::ExtHostReq::RunMacros(ref rm @ api::RunMacros{ ref run_id, ref query }) => {
+            hand.handle(rm, &macro_recur(*run_id, mtreev_from_api(query)).map(|x| mtreev_to_api(&x)))
+          }
         },
       ),
-      systems: eh.systems.into_iter().map(|decl| SystemCtor { decl, ext: weak.clone() }).collect(),
     });
     let weak = Arc::downgrade(&ret);
     thread::Builder::new()
@@ -263,7 +259,11 @@ impl SystemCtor {
       id,
     }));
     let root = (sys_inst.const_root.into_iter())
-      .map(|(k, v)| Member::from_api(api::Member { exported: true, name: k, kind: v }, &data))
+      .map(|(k, v)| Member::from_api(
+        api::Member { name: k, kind: v },
+        Substack::Bottom.push(deintern(k)),
+        &data
+      ))
       .collect_vec();
     data.0.const_root.set(root).unwrap();
     inst_g.insert(id, data.clone());
@@ -281,7 +281,7 @@ pub struct ReqPair<R: Request>(R, pub SyncSender<R::Response>);
 
 #[derive(destructure)]
 pub struct SystemInstData {
-  exprs: RwLock<HashMap<api::ExprTicket, (AtomicU32, RtExpr)>>,
+  exprs: RwLock<HashMap<api::ExprTicket, (AtomicU32, Expr)>>,
   ext: Extension,
   decl_id: api::SysDeclId,
   lex_filter: api::CharFilter,
@@ -303,11 +303,7 @@ impl System {
   pub fn id(&self) -> api::SysId { self.id }
   fn resolve(id: api::SysId) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
   fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.ext.0.reqnot }
-  fn give_expr(
+  fn give_expr(&self, ticket: api::ExprTicket, get_expr: impl FnOnce() -> Expr) -> api::ExprTicket {
-    &self,
-    ticket: api::ExprTicket,
-    get_expr: impl FnOnce() -> RtExpr,
-  ) -> api::ExprTicket {
     match self.0.exprs.write().unwrap().entry(ticket) {
       Entry::Occupied(mut oe) => {
         oe.get_mut().0.fetch_add(1, Ordering::Relaxed);
@@ -356,12 +352,22 @@ impl System {
   pub fn line_types(&self) -> impl Iterator<Item = Tok<String>> + '_ {
     self.line_types.iter().cloned()
   }
-  pub fn parse(&self, line: Vec<ParsTokTree>) -> OrcRes<Vec<ParsTokTree>> {
+  pub fn parse(
+    &self,
+    line: Vec<ParsTokTree>,
+    exported: bool,
+    comments: Vec<Comment>,
+  ) -> OrcRes<Vec<ParsTokTree>> {
     let line = line.iter().map(|t| t.to_api(&mut |n, _| match *n {})).collect_vec();
-    let parsed = (self.reqnot().request(api::ParseLine { sys: self.id(), line }))
+    let comments = comments.iter().map(Comment::to_api).collect_vec();
-      .map_err(|e| errv_from_apiv(e.iter()))?;
+    let parsed =
+      (self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line }))
+        .map_err(|e| OrcErrv::from_api(&e))?;
     Ok(ttv_from_api(parsed, &mut ()))
   }
+  pub fn request(&self, req: Vec<u8>) -> Vec<u8> {
+    self.reqnot().request(api::SysFwded(self.id(), req))
+  }
 }
 impl fmt::Debug for System {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

orchid-host/src/lex.rs

@@ -2,12 +2,14 @@ use std::num::NonZeroU64;
 use std::sync::Arc;
 
 use hashbrown::HashMap;
-use orchid_base::error::{mk_err, OrcErr, OrcRes};
+use orchid_base::error::{mk_errv, OrcErrv, OrcRes};
 use orchid_base::intern;
 use orchid_base::interner::{deintern, intern, Tok};
 use orchid_base::location::Pos;
+use orchid_base::number::{num_to_err, parse_num};
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
+use orchid_base::tree::Ph;
 
 use crate::api;
 use crate::extension::{AtomHand, System};
@@ -81,11 +83,9 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
     ParsTok::NS
   } else if ctx.strip_prefix("--[") {
     let (cmt, tail) = ctx.tail.split_once("]--").ok_or_else(|| {
-      vec![mk_err(
+      mk_errv(intern!(str: "Unterminated block comment"), "This block comment has no ending ]--", [
-        intern!(str: "Unterminated block comment"),
+        Pos::Range(start..start + 3).into(),
-        "This block comment has no ending ]--",
+      ])
-        [Pos::Range(start..start + 3).into()],
-      )]
     })?;
     ctx.set_tail(tail);
     ParsTok::Comment(Arc::new(cmt.to_string()))
@@ -98,11 +98,11 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
     ctx.trim_ws();
     while !ctx.strip_char('.') {
       if ctx.tail.is_empty() {
-        return Err(vec![mk_err(
+        return Err(mk_errv(
           intern!(str: "Unclosed lambda"),
           "Lambdae started with \\ should separate arguments from body with .",
           [Pos::Range(start..start + 1).into()],
-        )]);
+        ));
       }
       arg.push(lex_once(ctx)?);
       ctx.trim_ws();
@@ -113,33 +113,46 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
     ctx.trim_ws();
     while !ctx.strip_char(*rp) {
       if ctx.tail.is_empty() {
-        return Err(vec![mk_err(
+        return Err(mk_errv(
           intern!(str: "unclosed paren"),
           format!("this {lp} has no matching {rp}"),
           [Pos::Range(start..start + 1).into()],
-        )]);
+        ));
       }
       body.push(lex_once(ctx)?);
       ctx.trim_ws();
     }
     ParsTok::S(paren.clone(), body)
+  } else if ctx.strip_prefix("macro") &&
+    !ctx.tail.chars().next().is_some_and(|x| x.is_ascii_alphabetic())
+  {
+    ctx.strip_prefix("macro");
+    if ctx.strip_char('(') {
+      let pos = ctx.get_pos();
+      let numstr = ctx.get_start_matches(|x| x != ')').trim();
+      let num = parse_num(numstr).map_err(|e| num_to_err(e, pos))?;
+      ParsTok::Macro(Some(num.to_f64()))
+    } else {
+      ParsTok::Macro(None)
+    }
   } else {
     for sys in ctx.systems {
       let mut errors = Vec::new();
       if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-        let lexed = sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| {
+        let lx =
-          let mut sub_ctx = ctx.push(pos);
+          sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| match lex_once(&mut ctx.push(pos)) {
-          let ott =
+            Ok(t) => Some(api::SubLexed { pos, ticket: ctx.add_subtree(t) }),
-            lex_once(&mut sub_ctx).inspect_err(|e| errors.extend(e.iter().cloned())).ok()?;
+            Err(e) => {
-          Some(api::SubLexed { pos: sub_ctx.get_pos(), ticket: sub_ctx.add_subtree(ott) })
+              errors.push(e);
+              None
+            },
           });
-        match lexed {
+        match lx {
-          Ok(None) if errors.is_empty() => continue,
+          Err(e) => return Err(errors.into_iter().fold(OrcErrv::from_api(&e), |a, b| a + b)),
-          Ok(None) => return Err(errors),
+          Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos))),
-          Err(e) => return Err(e.into_iter().map(|e| OrcErr::from_api(&e)).collect()),
+          Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
-          Ok(Some(lexed)) => {
+            Some(errors) => return Err(errors),
-            ctx.set_pos(lexed.pos);
+            None => continue,
-            return Ok(tt_to_owned(&lexed.expr, ctx));
           },
         }
       }
@@ -149,11 +162,11 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
     } else if ctx.tail.starts_with(op_char) {
       ParsTok::Name(intern(ctx.get_start_matches(op_char)))
     } else {
-      return Err(vec![mk_err(
+      return Err(mk_errv(
         intern!(str: "Unrecognized character"),
         "The following syntax is meaningless.",
         [Pos::Range(start..start + 1).into()],
-      )]);
+      ));
     }
   };
   Ok(ParsTokTree { tok, range: start..ctx.get_pos() })
@@ -162,19 +175,28 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
   let tok = match &api.token {
     api::Token::Atom(atom) => ParsTok::Atom(AtomHand::from_api(atom.clone())),
-    api::Token::Bottom(err) => ParsTok::Bottom(err.iter().map(OrcErr::from_api).collect()),
+    api::Token::Bottom(err) => ParsTok::Bottom(OrcErrv::from_api(err)),
-    api::Token::Lambda(arg) =>
+    api::Token::LambdaHead(arg) => ParsTok::LambdaHead(ttv_to_owned(arg, ctx)),
-      ParsTok::LambdaHead(arg.iter().map(|t| tt_to_owned(t, ctx)).collect()),
+    api::Token::Lambda(arg, b) => ParsTok::Lambda(ttv_to_owned(arg, ctx), ttv_to_owned(b, ctx)),
     api::Token::Name(name) => ParsTok::Name(deintern(*name)),
     api::Token::S(p, b) => ParsTok::S(p.clone(), b.iter().map(|t| tt_to_owned(t, ctx)).collect()),
     api::Token::Slot(id) => return ctx.rm_subtree(*id),
     api::Token::BR => ParsTok::BR,
     api::Token::NS => ParsTok::NS,
     api::Token::Comment(c) => ParsTok::Comment(c.clone()),
+    api::Token::Ph(ph) => ParsTok::Ph(Ph::from_api(ph)),
+    api::Token::Macro(prio) => ParsTok::Macro(*prio)
   };
   ParsTokTree { range: api.range.clone(), tok }
 }
 
+fn ttv_to_owned<'a>(
+  api: impl IntoIterator<Item = &'a api::TokenTree>,
+  ctx: &mut LexCtx<'_>
+) -> Vec<ParsTokTree> {
+  api.into_iter().map(|t| tt_to_owned(t, ctx)).collect()
+}
+
 pub fn lex(text: Tok<String>, systems: &[System]) -> OrcRes<Vec<ParsTokTree>> {
   let mut sub_trees = HashMap::new();
   let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems };

orchid-host/src/lib.rs

@@ -7,3 +7,4 @@ pub mod lex;
 pub mod parse;
 pub mod subprocess;
 pub mod tree;
+pub mod macros;

orchid-host/src/macros.rs

@@ -0,0 +1,20 @@
+use std::sync::RwLock;
+
+use hashbrown::HashMap;
+use lazy_static::lazy_static;
+use orchid_base::macros::MTree;
+use trait_set::trait_set;
+use crate::api::ParsId;
+
+trait_set!{
+  trait MacroCB = Fn(Vec<MTree>) -> Option<Vec<MTree>> + Send + Sync;
+}
+
+lazy_static!{
+  static ref RECURSION: RwLock<HashMap<ParsId, Box<dyn MacroCB>>> = RwLock::default();
+}
+
+pub fn macro_recur(run_id: ParsId, input: Vec<MTree>) -> Option<Vec<MTree>> {
+  (RECURSION.read().unwrap()[&run_id])(input)
+}
+

orchid-host/src/parse.rs

@@ -2,18 +2,21 @@ use std::{iter, thread};
 
 use itertools::Itertools;
 use never::Never;
-use orchid_base::error::{mk_err, OrcErr, OrcRes, Reporter};
+use orchid_base::error::{mk_err, mk_errv, OrcErrv, OrcRes, Reporter};
 use orchid_base::intern;
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
+use orchid_base::macros::{MTok, MTree};
+use orchid_base::name::Sym;
 use orchid_base::parse::{
-  expect_end, line_items, parse_multiname, strip_fluff, try_pop_no_fluff, Comment, CompName,
+  expect_end, line_items, parse_multiname, strip_fluff, try_pop_no_fluff, Comment, Import,
-  Snippet,
+  Parsed, Snippet,
 };
 use orchid_base::tree::{Paren, TokTree, Token};
+use substack::Substack;
 
 use crate::extension::{AtomHand, System};
-use crate::tree::{Item, ItemKind, Member, MemberKind, Module, ParsTokTree};
+use crate::tree::{Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, ParsTokTree, Rule, RuleKind};
 
 type ParsSnippet<'a> = Snippet<'a, 'static, AtomHand, Never>;
 
@@ -22,15 +25,20 @@ pub trait ParseCtx: Send + Sync {
   fn reporter(&self) -> &impl Reporter;
 }
 
-pub fn parse_items(ctx: &impl ParseCtx, items: ParsSnippet) -> OrcRes<Vec<Item>> {
+pub fn parse_items(
+  ctx: &impl ParseCtx,
+  path: Substack<Tok<String>>,
+  items: ParsSnippet
+) -> OrcRes<Vec<Item>> {
   let lines = line_items(items);
   let mut ok = iter::from_fn(|| None).take(lines.len()).collect_vec();
   thread::scope(|s| {
     let mut threads = Vec::new();
-    for (slot, (cmts, item)) in ok.iter_mut().zip(lines.into_iter()) {
+    for (slot, Parsed { output: cmts, tail }) in ok.iter_mut().zip(lines.into_iter()) {
+      let path = &path;
       threads.push(s.spawn(move || {
-        *slot = Some(parse_item(ctx, cmts, item)?);
+        *slot = Some(parse_item(ctx, path.clone(), cmts, tail)?);
-        Ok::<(), Vec<OrcErr>>(())
+        Ok::<(), OrcErrv>(())
       }))
     }
     for t in threads {
@@ -42,136 +50,239 @@ pub fn parse_items(ctx: &impl ParseCtx, items: ParsSnippet) -> OrcRes<Vec<Item>>
 
 pub fn parse_item(
   ctx: &impl ParseCtx,
+  path: Substack<Tok<String>>,
   comments: Vec<Comment>,
   item: ParsSnippet,
 ) -> OrcRes<Vec<Item>> {
   match item.pop_front() {
     Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
       n if *n == intern!(str: "export") => match try_pop_no_fluff(postdisc)? {
-        (TokTree { tok: Token::Name(n), .. }, postdisc) =>
+        Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
-          parse_item_2(ctx, comments, true, n.clone(), postdisc),
+          parse_exportable_item(ctx, path, comments, true, n.clone(), tail),
-        (TokTree { tok: Token::NS, .. }, postdisc) => {
+        Parsed { output: TokTree { tok: Token::NS, .. }, tail } => {
-          let (exports, surplus) = parse_multiname(ctx.reporter(), postdisc)?;
+          let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail)?;
           let mut ok = Vec::new();
-          exports.into_iter().for_each(|e| match (&e.path.as_slice(), e.name) {
+          exports.into_iter().for_each(|(e, pos)| match (&e.path.as_slice(), e.name) {
-            ([], Some(n)) => ok.push(Item {
+            ([], Some(n)) =>
-              comments: comments.clone(),
+              ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
-              pos: e.pos.clone(),
-              kind: ItemKind::Export(n),
-            }),
             (_, Some(_)) => ctx.reporter().report(mk_err(
               intern!(str: "Compound export"),
               "Cannot export compound names (names containing the :: separator)",
-              [e.pos.into()],
+              [pos.into()],
             )),
             (_, None) => ctx.reporter().report(mk_err(
               intern!(str: "Wildcard export"),
               "Exports cannot contain the globstar *",
-              [e.pos.into()],
+              [pos.into()],
             )),
           });
-          expect_end(surplus)?;
+          expect_end(tail)?;
           Ok(ok)
         },
-        (bogus, _) => Err(vec![mk_err(
+        Parsed { output, .. } => Err(mk_errv(
           intern!(str: "Malformed export"),
           "`export` can either prefix other lines or list names inside ::( ) or ::[ ]",
-          [Pos::Range(bogus.range.clone()).into()],
+          [Pos::Range(output.range.clone()).into()],
-        )]),
+        )),
       },
       n if *n == intern!(str: "import") => parse_import(ctx, postdisc).map(|v| {
-        Vec::from_iter(v.into_iter().map(|t| Item {
+        Vec::from_iter(v.into_iter().map(|(t, pos)| Item {
           comments: comments.clone(),
-          pos: Pos::Range(postdisc.pos()),
+          pos,
           kind: ItemKind::Import(t),
         }))
       }),
-      n => parse_item_2(ctx, comments, false, n.clone(), postdisc),
+      n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc),
     },
-    Some(_) => Err(vec![mk_err(
+    Some(_) =>
-      intern!(str: "Expected a line type"),
+      Err(mk_errv(intern!(str: "Expected a line type"), "All lines must begin with a keyword", [
-      "All lines must begin with a keyword",
+        Pos::Range(item.pos()).into(),
-      [Pos::Range(item.pos()).into()],
+      ])),
-    )]),
     None => unreachable!("These lines are filtered and aggregated in earlier stages"),
   }
 }
 
-pub fn parse_import(ctx: &impl ParseCtx, tail: ParsSnippet) -> OrcRes<Vec<CompName>> {
+pub fn parse_import(ctx: &impl ParseCtx, tail: ParsSnippet) -> OrcRes<Vec<(Import, Pos)>> {
-  let (imports, surplus) = parse_multiname(ctx.reporter(), tail)?;
+  let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail)?;
-  expect_end(surplus)?;
+  expect_end(tail)?;
   Ok(imports)
 }
 
-pub fn parse_item_2(
+pub fn parse_exportable_item(
   ctx: &impl ParseCtx,
+  path: Substack<Tok<String>>,
   comments: Vec<Comment>,
   exported: bool,
   discr: Tok<String>,
   tail: ParsSnippet,
 ) -> OrcRes<Vec<Item>> {
   let kind = if discr == intern!(str: "mod") {
-    let (name, body) = parse_module(ctx, tail)?;
+    let (name, body) = parse_module(ctx, path, tail)?;
-    ItemKind::Member(Member::new(exported, name, MemberKind::Mod(body)))
+    ItemKind::Member(Member::new(name, MemberKind::Mod(body)))
   } else if discr == intern!(str: "const") {
     let (name, val) = parse_const(tail)?;
-    ItemKind::Member(Member::new(exported, name, MemberKind::Const(val)))
+    let locator = CodeLocator::to_const(path.push(name.clone()).unreverse());
+    ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val))))
   } else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
-    let line = sys.parse(tail.to_vec())?;
+    let line = sys.parse(tail.to_vec(), exported, comments)?;
-    return parse_items(ctx, Snippet::new(tail.prev(), &line));
+    return parse_items(ctx, path, Snippet::new(tail.prev(), &line));
   } else {
     let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
-    return Err(vec![mk_err(
+    return Err(mk_errv(
       intern!(str: "Unrecognized line type"),
       format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
       [Pos::Range(tail.prev().range.clone()).into()],
-    )]);
+    ));
   };
   Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }])
 }
 
-pub fn parse_module(ctx: &impl ParseCtx, tail: ParsSnippet) -> OrcRes<(Tok<String>, Module)> {
+pub fn parse_module(
+  ctx: &impl ParseCtx,
+  path: Substack<Tok<String>>,
+  tail: ParsSnippet
+) -> OrcRes<(Tok<String>, Module)> {
   let (name, tail) = match try_pop_no_fluff(tail)? {
-    (TokTree { tok: Token::Name(n), .. }, tail) => (n.clone(), tail),
+    Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
-    (tt, _) =>
+    Parsed { output, .. } =>
-      return Err(vec![mk_err(
+      return Err(mk_errv(
         intern!(str: "Missing module name"),
-        format!("A name was expected, {tt} was found"),
+        format!("A name was expected, {output} was found"),
-        [Pos::Range(tt.range.clone()).into()],
+        [Pos::Range(output.range.clone()).into()],
-      )]),
+      )),
   };
-  let (body, surplus) = match try_pop_no_fluff(tail)? {
+  let Parsed { output, tail: surplus } = try_pop_no_fluff(tail)?;
-    (TokTree { tok: Token::S(Paren::Round, b), .. }, tail) => (b, tail),
+  expect_end(surplus)?;
-    (tt, _) =>
+  let body = output.as_s(Paren::Round).ok_or_else(|| mk_errv(
-      return Err(vec![mk_err(
     intern!(str: "Expected module body"),
-        format!("A ( block ) was expected, {tt} was found"),
+    format!("A ( block ) was expected, {output} was found"),
-        [Pos::Range(tt.range.clone()).into()],
+    [Pos::Range(output.range.clone()).into()],
-      )]),
+  ))?;
-  };
+  let path = path.push(name.clone());
-  let items = parse_items(ctx, ParsSnippet::new(surplus.prev(), body))?;
+  Ok((name, Module::new(parse_items(ctx, path, body)?)))
-  Ok((name, Module { imports: vec![], items }))
 }
 
 pub fn parse_const(tail: ParsSnippet) -> OrcRes<(Tok<String>, Vec<ParsTokTree>)> {
-  let (name, tail) = match try_pop_no_fluff(tail)? {
+  let Parsed { output, tail } = try_pop_no_fluff(tail)?;
-    (TokTree { tok: Token::Name(n), .. }, tail) => (n.clone(), tail),
+  let name = output.as_name().ok_or_else(|| mk_errv(
-    (tt, _) =>
-      return Err(vec![mk_err(
     intern!(str: "Missing module name"),
-        format!("A name was expected, {tt} was found"),
+    format!("A name was expected, {output} was found"),
-        [Pos::Range(tt.range.clone()).into()],
+    [Pos::Range(output.range.clone()).into()],
-      )]),
+  ))?;
-  };
+  let Parsed { output, tail } = try_pop_no_fluff(tail)?;
-  let tail = match try_pop_no_fluff(tail)? {
+  if !output.is_kw(intern!(str: "=")) {
-    (TokTree { tok: Token::Name(n), .. }, tail) if *n == intern!(str: ":=") => tail,
+    return Err(mk_errv(
-    (tt, _) =>
-      return Err(vec![mk_err(
       intern!(str: "Missing walrus := separator"),
-        format!("Expected operator := , found {tt}"),
+      format!("Expected operator := , found {output}"),
-        [Pos::Range(tt.range.clone()).into()],
+      [Pos::Range(output.range.clone()).into()],
-      )]),
+    ))
-  };
+  }
   try_pop_no_fluff(tail)?;
   Ok((name, tail.iter().flat_map(strip_fluff).collect_vec()))
 }
+
+pub fn parse_mtree<'a>(
+  mut snip: ParsSnippet<'a>
+) -> OrcRes<Vec<MTree<'static>>> {
+  let mut mtreev = Vec::new();
+  while let Some((ttree, tail)) = snip.pop_front() {
+    let (range, tok, tail) = match &ttree.tok {
+      Token::S(p, b) => (
+        ttree.range.clone(),
+        MTok::S(*p, parse_mtree(Snippet::new(ttree, b))?),
+        tail,
+      ),
+      Token::Name(tok) => {
+        let mut segments = vec![tok.clone()];
+        let mut end = ttree.range.end;
+        while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() {
+          let Parsed { output, tail } = try_pop_no_fluff(tail)?;
+          segments.push(output.as_name().ok_or_else(|| mk_errv(
+            intern!(str: "Namespaced name interrupted"),
+            "In expression context, :: must always be followed by a name.\n\
+            ::() is permitted only in import and export items", 
+            [Pos::Range(output.range.clone()).into()]
+          ))?);
+          snip = tail;
+          end = output.range.end;
+        }
+        (ttree.range.start..end, MTok::Name(Sym::new(segments).unwrap()), snip)
+      },
+      Token::NS => return Err(mk_errv(
+        intern!(str: "Unexpected :: in macro pattern"),
+        ":: can only follow a name outside export statements",
+        [Pos::Range(ttree.range.clone()).into()]
+      )),
+      Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail),
+      Token::Atom(_) | Token::Macro(_) => return Err(mk_errv(
+        intern!(str: "Unsupported token in macro patterns"), 
+        format!("Macro patterns can only contain names, braces, and lambda, not {ttree}."),
+        [Pos::Range(ttree.range.clone()).into()]
+      )),
+      Token::BR | Token::Comment(_) => continue,
+      Token::Bottom(e) => return Err(e.clone()),
+      Token::Lambda(arg, body) => {
+        let tok = MTok::Lambda(
+          parse_mtree(Snippet::new(&ttree, &arg))?,
+          parse_mtree(Snippet::new(&ttree, &body))?,
+        );
+        (ttree.range.clone(), tok, tail)
+      },
+      Token::LambdaHead(arg) => (
+        ttree.range.start..snip.pos().end,
+        MTok::Lambda(parse_mtree(Snippet::new(&ttree, &arg))?, parse_mtree(tail)?),
+        Snippet::new(ttree, &[]),
+      ),
+      Token::Slot(_) | Token::X(_) => panic!("Did not expect {} in parsed token tree", &ttree.tok),
+    };
+    mtreev.push(MTree { pos: Pos::Range(range.clone()), tok });
+    snip = tail;
+  }
+  Ok(mtreev)
+}
+
+pub fn parse_macro(tail: ParsSnippet, macro_i: u16, path: Substack<Tok<String>>) -> OrcRes<Vec<Rule>> {
+  let (surplus, prev, block) = match try_pop_no_fluff(tail)? {
+    Parsed { tail, output: o@TokTree { tok: Token::S(Paren::Round, b), .. } } => (tail, o, b),
+    Parsed { output, .. } => return Err(mk_errv(
+      intern!(str: "m"),
+      format!("Macro blocks must either start with a block or a ..$:number"), 
+      [Pos::Range(output.range.clone()).into()]
+    )),
+  };
+  expect_end(surplus)?;
+  let mut errors = Vec::new();
+  let mut rules = Vec::new();
+  for (i, item) in line_items(Snippet::new(prev, &block)).into_iter().enumerate() {
+    let Parsed { tail, output } = try_pop_no_fluff(item.tail)?;
+    if !output.is_kw(intern!(str: "rule")) {
+      errors.extend(mk_errv(
+        intern!(str: "non-rule in macro"), 
+        format!("Expected `rule`, got {output}"),
+        [Pos::Range(output.range.clone()).into()]
+      ));
+      continue
+    };
+    let (pat, body) = match tail.split_once(|t| t.is_kw(intern!(str: "=>"))) {
+      Some((a, b)) => (a, b),
+      None => {
+        errors.extend(mk_errv(
+          intern!(str: "no => in macro rule"),
+          "The pattern and body of a rule must be separated by a =>",
+          [Pos::Range(tail.pos()).into()],
+        ));
+        continue
+      }
+    };
+    rules.push(Rule {
+      comments: item.output,
+      pos: Pos::Range(tail.pos()),
+      pattern: parse_mtree(pat)?,
+      kind: RuleKind::Native(Code::from_code(
+        CodeLocator::to_rule(path.unreverse(), macro_i, i as u16),
+        body.to_vec(),
+      ))
+    })
+  }
+  if let Ok(e) = OrcErrv::new(errors) { Err(e) } else { Ok(rules) }
+}

orchid-host/src/subprocess.rs

@@ -1,17 +1,21 @@
-use std::io::{self, BufRead as _};
+use std::io::{self, BufRead as _, Write};
 use std::path::PathBuf;
 use std::sync::Mutex;
 use std::{process, thread};
 
+use orchid_api::ExtensionHeader;
+use orchid_api_traits::{Decode, Encode};
 use orchid_base::logging::Logger;
 use orchid_base::msg::{recv_msg, send_msg};
 
+use crate::api;
 use crate::extension::ExtensionPort;
 
 pub struct Subprocess {
   child: Mutex<process::Child>,
   stdin: Mutex<process::ChildStdin>,
   stdout: Mutex<process::ChildStdout>,
+  header: ExtensionHeader,
 }
 impl Subprocess {
   pub fn new(mut cmd: process::Command, logger: Logger) -> io::Result<Self> {
@@ -22,8 +26,11 @@ impl Subprocess {
       .stdout(process::Stdio::piped())
       .stderr(process::Stdio::piped())
       .spawn()?;
-    let stdin = child.stdin.take().unwrap();
+    let mut stdin = child.stdin.take().unwrap();
-    let stdout = child.stdout.take().unwrap();
+    api::HostHeader { log_strategy: logger.strat() }.encode(&mut stdin);
+    stdin.flush()?;
+    let mut stdout = child.stdout.take().unwrap();
+    let header = ExtensionHeader::decode(&mut stdout);
     let child_stderr = child.stderr.take().unwrap();
     thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || {
       let mut reader = io::BufReader::new(child_stderr);
@@ -35,14 +42,25 @@ impl Subprocess {
         logger.log(buf);
       }
     })?;
-    Ok(Self { child: Mutex::new(child), stdin: Mutex::new(stdin), stdout: Mutex::new(stdout) })
+    Ok(Self {
+      child: Mutex::new(child),
+      stdin: Mutex::new(stdin),
+      stdout: Mutex::new(stdout),
+      header,
+    })
   }
 }
 impl Drop for Subprocess {
   fn drop(&mut self) { self.child.lock().unwrap().wait().expect("Extension exited with error"); }
 }
 impl ExtensionPort for Subprocess {
-  fn send(&self, msg: &[u8]) { send_msg(&mut *self.stdin.lock().unwrap(), msg).unwrap() }
+  fn header(&self) -> &orchid_api::ExtensionHeader { &self.header }
+  fn send(&self, msg: &[u8]) {
+    if msg.starts_with(&[0, 0, 0, 0x1c]) {
+      panic!("Received unnecessary prefix");
+    }
+    send_msg(&mut *self.stdin.lock().unwrap(), msg).unwrap()
+  }
   fn receive(&self) -> Option<Vec<u8>> {
     match recv_msg(&mut *self.stdout.lock().unwrap()) {
       Ok(msg) => Some(msg),

orchid-host/src/tree.rs

@@ -1,16 +1,20 @@
+use std::fmt::Debug;
 use std::sync::{Mutex, OnceLock};
 
 use itertools::Itertools;
 use never::Never;
 use orchid_base::error::OrcRes;
-use orchid_base::interner::{deintern, Tok};
+use orchid_base::interner::{deintern, intern, Tok};
 use orchid_base::location::Pos;
+use orchid_base::macros::{mtreev_from_api, MTree};
 use orchid_base::name::Sym;
-use orchid_base::parse::{Comment, CompName};
+use orchid_base::parse::{Comment, Import};
-use orchid_base::tree::{ttv_from_api, TokTree, Token};
+use orchid_base::tree::{TokTree, Token};
+use ordered_float::NotNan;
+use substack::{with_iter_stack, Substack};
 
 use crate::api;
-use crate::expr::RtExpr;
+use crate::expr::Expr;
 use crate::extension::{AtomHand, System};
 
 pub type ParsTokTree = TokTree<'static, AtomHand, Never>;
@@ -25,81 +29,165 @@ pub struct Item {
 
 #[derive(Debug)]
 pub enum ItemKind {
-  Raw(Vec<ParsTokTree>),
   Member(Member),
   Export(Tok<String>),
-  Import(CompName),
+  Import(Import),
+  Macro(Option<NotNan<f64>>, Vec<Rule>)
 }
 
 impl Item {
-  pub fn from_api(tree: api::Item, sys: &System) -> Self {
+  pub fn from_api<'a>(
+    tree: api::Item,
+    path: Substack<Tok<String>>,
+    sys: &System
+  ) -> Self {
     let kind = match tree.kind {
-      api::ItemKind::Raw(tokv) => ItemKind::Raw(ttv_from_api(tokv, &mut ())),
+      api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys)),
-      api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, sys)),
+      api::ItemKind::Import(i) =>
-      api::ItemKind::Import(i) => ItemKind::Import(CompName::from_api(i)),
+        ItemKind::Import(Import{ path: Sym::deintern(i).iter().collect(), name: None }),
       api::ItemKind::Export(e) => ItemKind::Export(deintern(e)),
+      api::ItemKind::Macro(api::MacroBlock { priority, rules }) => ItemKind::Macro(priority, {
+        Vec::from_iter(rules.into_iter().map(|api| Rule {
+          pos: Pos::from_api(&api.location),
+          pattern: mtreev_from_api(&api.pattern),
+          kind: RuleKind::Remote(sys.clone(), api.id),
+          comments: api.comments.iter().map(Comment::from_api).collect_vec()
+        }))
+      })
     };
-    let comments = tree
+    let comments = tree.comments.iter().map(Comment::from_api).collect_vec();
-      .comments
-      .into_iter()
-      .map(|(text, l)| Comment { text, pos: Pos::from_api(&l) })
-      .collect_vec();
     Self { pos: Pos::from_api(&tree.location), comments, kind }
   }
 }
 
 #[derive(Debug)]
 pub struct Member {
-  pub exported: bool,
   pub name: Tok<String>,
   pub kind: OnceLock<MemberKind>,
   pub lazy: Mutex<Option<LazyMemberHandle>>,
 }
 impl Member {
-  pub fn from_api(api::Member { exported: public, name, kind }: api::Member, sys: &System) -> Self {
+  pub fn from_api<'a>(
-    let (kind, lazy) = match kind {
+    api: api::Member,
-      api::MemberKind::Const(c) =>
+    path: Substack<Tok<String>>,
-        (OnceLock::from(MemberKind::PreCnst(RtExpr::from_api(c, sys))), None),
+    sys: &System,
-      api::MemberKind::Module(m) =>
+  ) -> Self {
-        (OnceLock::from(MemberKind::Mod(Module::from_api(m, sys))), None),
+    let name = deintern(api.name);
-      api::MemberKind::Lazy(id) => (OnceLock::new(), Some(LazyMemberHandle(id, sys.clone()))),
+    let full_path = path.push(name.clone());
+    let kind = match api.kind {
+      api::MemberKind::Lazy(id) =>
+        return LazyMemberHandle(id, sys.clone(), intern(&full_path.unreverse())).to_member(name),
+      api::MemberKind::Const(c) => MemberKind::Const(Code::from_expr(
+        CodeLocator::to_const(full_path.unreverse()),
+        Expr::from_api(c, &mut ())
+      )),
+      api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, full_path, sys)),
     };
-    Member { exported: public, name: deintern(name), kind, lazy: Mutex::new(lazy) }
+    Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() }
   }
-  pub fn new(public: bool, name: Tok<String>, kind: MemberKind) -> Self {
+  pub fn new(name: Tok<String>, kind: MemberKind) -> Self {
-    Member { exported: public, name, kind: OnceLock::from(kind), lazy: Mutex::default() }
+    Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() }
   }
 }
 
 #[derive(Debug)]
 pub enum MemberKind {
-  Const(Vec<ParsTokTree>),
+  Const(Code),
-  PreCnst(RtExpr),
   Mod(Module),
 }
 
 #[derive(Debug)]
 pub struct Module {
   pub imports: Vec<Sym>,
+  pub exports: Vec<Tok<String>>,
   pub items: Vec<Item>,
 }
 impl Module {
-  pub fn from_api(m: api::Module, sys: &System) -> Self {
+  pub fn new(items: impl IntoIterator<Item = Item>) -> Self {
-    Self {
+    let items = items.into_iter().collect_vec();
-      imports: m.imports.into_iter().map(|m| Sym::from_tok(deintern(m)).unwrap()).collect_vec(),
+    let exports = (items.iter())
-      items: m.items.into_iter().map(|i| Item::from_api(i, sys)).collect_vec(),
+      .filter_map(|i| match &i.kind {
+        ItemKind::Export(e) => Some(e.clone()),
+        _ => None,
+      })
+      .collect_vec();
+    Self { imports: vec![], exports, items }
   }
+  pub fn from_api(m: api::Module, path: Substack<Tok<String>>, sys: &System) -> Self {
+    let mut output = Vec::new();
+    for item in m.items.into_iter() {
+      let next = Item::from_api(item, path.clone(), sys);
+      output.push(next);
+    }
+    Self::new(output)
   }
 }
 
 #[derive(Debug)]
-pub struct LazyMemberHandle(api::TreeId, System);
+pub struct LazyMemberHandle(api::TreeId, System, Tok<Vec<Tok<String>>>);
 impl LazyMemberHandle {
   pub fn run(self) -> OrcRes<MemberKind> {
     match self.1.get_tree(self.0) {
-      api::MemberKind::Const(c) => Ok(MemberKind::PreCnst(RtExpr::from_api(c, &self.1))),
+      api::MemberKind::Const(c) => Ok(MemberKind::Const(Code {
-      api::MemberKind::Module(m) => Ok(MemberKind::Mod(Module::from_api(m, &self.1))),
+        bytecode: Expr::from_api(c, &mut ()).into(),
-      api::MemberKind::Lazy(id) => Self(id, self.1).run(),
+        locator: CodeLocator { steps: self.2, rule_loc: None },
+        source: None,
+      })),
+      api::MemberKind::Module(m) => with_iter_stack(self.2.iter().cloned(), |path| {
+        Ok(MemberKind::Mod(Module::from_api(m, path, &self.1)))
+      }),
+      api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run(),
     }
   }
+  pub fn to_member(self, name: Tok<String>) -> Member {
+    Member { name, kind: OnceLock::new(), lazy: Mutex::new(Some(self)) }
+  }
+}
+
+#[derive(Debug)]
+pub struct Rule {
+  pub pos: Pos,
+  pub comments: Vec<Comment>,
+  pub pattern: Vec<MTree<'static>>,
+  pub kind: RuleKind,
+}
+
+#[derive(Debug)]
+pub enum RuleKind {
+  Remote(System, api::MacroId),
+  Native(Code),
+}
+
+#[derive(Debug)]
+pub struct Code {
+  locator: CodeLocator,
+  source: Option<Vec<ParsTokTree>>,
+  bytecode: OnceLock<Expr>,
+}
+impl Code {
+  pub fn from_expr(locator: CodeLocator, expr: Expr) -> Self {
+    Self { locator, source: None, bytecode: expr.into() }
+  }
+  pub fn from_code(locator: CodeLocator, code: Vec<ParsTokTree>) -> Self {
+    Self { locator, source: Some(code), bytecode: OnceLock::new() }
+  }
+}
+
+/// Selects a code element
+/// 
+/// Either the steps point to a constant and rule_loc is None, or the steps point to a module and
+/// rule_loc selects a macro rule within that module
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct CodeLocator {
+  steps: Tok<Vec<Tok<String>>>,
+  /// Index of a macro block in the module demarked by the steps, and a rule in that macro
+  rule_loc: Option<(u16, u16)>,
+}
+impl CodeLocator {
+  pub fn to_const(path: impl IntoIterator<Item = Tok<String>>) -> Self {
+    Self { steps: intern(&path.into_iter().collect_vec()), rule_loc: None }
+  }
+  pub fn to_rule(path: impl IntoIterator<Item = Tok<String>>, macro_i: u16, rule_i: u16) -> Self {
+    Self { steps: intern(&path.into_iter().collect_vec()), rule_loc: Some((macro_i, rule_i)) }
+  }
 }

orchid-std/src/number/num_atom.rs

@@ -1,10 +1,9 @@
-use never::Never;
 use orchid_api_derive::Coding;
 use orchid_base::error::OrcRes;
-use orchid_extension::atom::{AtomFactory, Atomic, AtomicFeatures, ReqPck, ToAtom, TypAtom};
+use orchid_extension::atom::{AtomFactory, MethodSet, Atomic, AtomicFeatures, ToAtom, TypAtom};
 use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
 use orchid_extension::conv::TryFromExpr;
-use orchid_extension::expr::ExprHandle;
+use orchid_extension::expr::Expr;
 use ordered_float::NotNan;
 
 #[derive(Clone, Debug, Coding)]
@@ -12,13 +11,13 @@ pub struct Int(pub i64);
 impl Atomic for Int {
   type Variant = ThinVariant;
   type Data = Self;
-  type Req = Never;
+  fn reg_reqs() -> MethodSet<Self> {
+    MethodSet::new()
   }
-impl ThinAtom for Int {
-  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
 }
+impl ThinAtom for Int {}
 impl TryFromExpr for Int {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
+  fn try_from_expr(expr: Expr) -> OrcRes<Self> {
     TypAtom::<Int>::try_from_expr(expr).map(|t| t.value)
   }
 }
@@ -28,13 +27,11 @@ pub struct Float(pub NotNan<f64>);
 impl Atomic for Float {
   type Variant = ThinVariant;
   type Data = Self;
-  type Req = Never;
+  fn reg_reqs() -> MethodSet<Self> { MethodSet::new() }
-}
-impl ThinAtom for Float {
-  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
 }
+impl ThinAtom for Float {}
 impl TryFromExpr for Float {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
+  fn try_from_expr(expr: Expr) -> OrcRes<Self> {
     TypAtom::<Float>::try_from_expr(expr).map(|t| t.value)
   }
 }
@@ -44,10 +41,10 @@ pub enum Numeric {
   Float(NotNan<f64>),
 }
 impl TryFromExpr for Numeric {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
+  fn try_from_expr(expr: Expr) -> OrcRes<Self> {
-    Int::try_from_expr(expr.clone()).map(|t| Numeric::Int(t.0)).or_else(|e| {
+    Int::try_from_expr(expr.clone())
-      Float::try_from_expr(expr).map(|t| Numeric::Float(t.0)).map_err(|e2| [e, e2].concat())
+      .map(|t| Numeric::Int(t.0))
-    })
+      .or_else(|e| Float::try_from_expr(expr).map(|t| Numeric::Float(t.0)).map_err(|e2| e + e2))
   }
 }
 impl ToAtom for Numeric {

orchid-std/src/number/num_lexer.rs

@@ -20,7 +20,7 @@ impl Lexer for NumLexer {
       Ok(Numeric::Float(f)) => Float(f).factory(),
       Ok(Numeric::Uint(uint)) => Int(uint.try_into().unwrap()).factory(),
       Ok(Numeric::Decimal(dec)) => Float(NotNan::new(dec.try_into().unwrap()).unwrap()).factory(),
-      Err(e) => return Err(vec![num_to_err(e, ctx.pos(all))]),
+      Err(e) => return Err(num_to_err(e, ctx.pos(all)).into()),
     };
     Ok((tail, GenTok::X(fac).at(ctx.pos(all)..ctx.pos(tail))))
   }

orchid-std/src/std.rs

@@ -1,11 +1,13 @@
 use std::sync::Arc;
 
+use never::Never;
 use orchid_base::interner::Tok;
 use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
+use orchid_extension::entrypoint::ExtReq;
 use orchid_extension::fs::DeclFs;
 use orchid_extension::system::{System, SystemCard};
 use orchid_extension::system_ctor::SystemCtor;
-use orchid_extension::tree::{comments, fun, module, root_mod, GenMemberKind};
+use orchid_extension::tree::{comments, fun, module, root_mod, MemKind};
 
 use crate::number::num_atom::{Float, Int};
 use crate::string::str_atom::{IntStrAtom, StrAtom};
@@ -23,14 +25,17 @@ impl SystemCtor for StdSystem {
 }
 impl SystemCard for StdSystem {
   type Ctor = Self;
-  const ATOM_DEFS: &'static [Option<&'static dyn AtomDynfo>] =
+  type Req = Never;
-    &[Some(Int::INFO), Some(Float::INFO), Some(StrAtom::INFO), Some(IntStrAtom::INFO)];
+  fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
+    [Some(Int::dynfo()), Some(Float::dynfo()), Some(StrAtom::dynfo()), Some(IntStrAtom::dynfo())]
+  }
 }
 impl System for StdSystem {
+  fn request(_: ExtReq, req: Self::Req) -> orchid_base::reqnot::Receipt { match req {} }
   fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }
   fn parsers() -> Vec<orchid_extension::parser::ParserObj> { vec![] }
   fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
-  fn env() -> Vec<(Tok<String>, GenMemberKind)> {
+  fn env() -> Vec<(Tok<String>, MemKind)> {
     vec![root_mod("std", [], [module(true, "string", [], [comments(
       ["Concatenate two strings"],
       fun(true, "concat", |left: OrcString, right: OrcString| {

orchid-std/src/string/str_atom.rs

@@ -1,52 +1,53 @@
 use std::borrow::Cow;
 use std::io;
-use std::num::NonZeroU64;
+use std::ops::Deref;
 use std::sync::Arc;
 
-use never::Never;
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Encode, Request};
-use orchid_base::error::{mk_err, OrcRes};
+use orchid_base::error::{mk_errv, OrcRes};
-use orchid_base::id_store::IdStore;
 use orchid_base::intern;
 use orchid_base::interner::{deintern, intern, Tok};
-use orchid_extension::atom::{Atomic, ReqPck, TypAtom};
+use orchid_extension::atom::{AtomMethod, Atomic, MethodSet, Supports, TypAtom};
 use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use orchid_extension::conv::TryFromExpr;
-use orchid_extension::expr::ExprHandle;
+use orchid_extension::expr::Expr;
 use orchid_extension::system::SysCtx;
 
-pub static STR_REPO: IdStore<Arc<String>> = IdStore::new();
-
 #[derive(Copy, Clone, Coding)]
 pub struct StringGetVal;
 impl Request for StringGetVal {
-  type Response = String;
+  type Response = Arc<String>;
+}
+impl AtomMethod for StringGetVal {
+  const NAME: &str = "std::string_get_val";
+}
+impl Supports<StringGetVal> for StrAtom {
+  fn handle(&self, _: SysCtx, _: StringGetVal) -> <StringGetVal as Request>::Response {
+    self.0.clone()
+  }
 }
 
-pub struct StrAtom(NonZeroU64);
+#[derive(Clone)]
+pub struct StrAtom(Arc<String>);
 impl Atomic for StrAtom {
   type Variant = OwnedVariant;
-  type Data = NonZeroU64;
+  type Data = ();
-  type Req = StringGetVal;
+  fn reg_reqs() -> MethodSet<Self> { MethodSet::new().handle::<StringGetVal>() }
 }
 impl StrAtom {
-  pub fn new(str: Arc<String>) -> Self { Self(STR_REPO.add(str).id()) }
+  pub fn new(str: Arc<String>) -> Self { Self(str) }
+  pub fn value(&self) -> Arc<String> { self.0.clone() }
 }
-impl Clone for StrAtom {
+impl Deref for StrAtom {
-  fn clone(&self) -> Self { Self::new(self.local_value()) }
+  type Target = str;
-}
+  fn deref(&self) -> &Self::Target { &self.0 }
-impl StrAtom {
-  fn try_local_value(&self) -> Option<Arc<String>> { STR_REPO.get(self.0).map(|r| r.clone()) }
-  fn local_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
 }
 impl OwnedAtom for StrAtom {
   type Refs = ();
-  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0) }
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-  fn same(&self, _: SysCtx, other: &Self) -> bool { self.local_value() == other.local_value() }
-  fn handle_req(&self, pck: impl ReqPck<Self>) { self.local_value().encode(pck.unpack().1) }
   fn serialize(&self, _: SysCtx, sink: &mut (impl io::Write + ?Sized)) -> Self::Refs {
-    self.local_value().encode(sink)
+    self.deref().encode(sink)
   }
   fn deserialize(mut ctx: impl DeserializeCtx, _: Self::Refs) -> Self {
     Self::new(Arc::new(ctx.read::<String>()))
@@ -58,7 +59,7 @@ pub struct IntStrAtom(Tok<String>);
 impl Atomic for IntStrAtom {
   type Variant = OwnedVariant;
   type Data = orchid_api::TStr;
-  type Req = Never;
+  fn reg_reqs() -> MethodSet<Self> { MethodSet::new() }
 }
 impl From<Tok<String>> for IntStrAtom {
   fn from(value: Tok<String>) -> Self { Self(value) }
@@ -66,8 +67,7 @@ impl From<Tok<String>> for IntStrAtom {
 impl OwnedAtom for IntStrAtom {
   type Refs = ();
   fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.marker()) }
-  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
+  fn print(&self, _ctx: SysCtx) -> String { format!("{:?}i", *self.0) }
-  fn print(&self, _ctx: SysCtx) -> String { format!("{:?}i", self.0.as_str()) }
   fn serialize(&self, _: SysCtx, write: &mut (impl io::Write + ?Sized)) { self.0.encode(write) }
   fn deserialize(ctx: impl DeserializeCtx, _: ()) -> Self { Self(intern(&ctx.decode::<String>())) }
 }
@@ -81,22 +81,19 @@ impl<'a> OrcString<'a> {
   pub fn get_string(&self) -> Arc<String> {
     match &self {
       Self::Int(tok) => deintern(tok.value).arc(),
-      Self::Val(atom) => match STR_REPO.get(**atom) {
+      Self::Val(atom) => atom.request(StringGetVal),
-        Some(rec) => rec.clone(),
-        None => Arc::new(atom.request(StringGetVal)),
-      },
     }
   }
 }
 
 impl TryFromExpr for OrcString<'static> {
-  fn try_from_expr(expr: ExprHandle) -> OrcRes<OrcString<'static>> {
+  fn try_from_expr(expr: Expr) -> OrcRes<OrcString<'static>> {
-    if let Ok(v) = TypAtom::<StrAtom>::downcast(expr.clone()) {
+    if let Ok(v) = TypAtom::<StrAtom>::try_from_expr(expr.clone()) {
       return Ok(OrcString::Val(v));
     }
-    match TypAtom::<IntStrAtom>::downcast(expr) {
+    match TypAtom::<IntStrAtom>::try_from_expr(expr) {
       Ok(t) => Ok(OrcString::Int(t)),
-      Err(e) => Err(vec![mk_err(intern!(str: "A string was expected"), "", [e.0.clone().into()])]),
+      Err(e) => Err(mk_errv(intern!(str: "A string was expected"), "", e.pos_iter())),
     }
   }
 }

orchid-std/src/string/str_lexer.rs

@@ -1,11 +1,11 @@
 use itertools::Itertools;
-use orchid_base::error::{mk_err, OrcErr, OrcRes};
+use orchid_base::error::{mk_err, mk_errv, OrcErr, OrcRes};
 use orchid_base::interner::intern;
 use orchid_base::location::Pos;
 use orchid_base::tree::{vname_tv, wrap_tokv};
 use orchid_base::{intern, vname};
 use orchid_extension::atom::AtomicFeatures;
-use orchid_extension::lexer::{err_lexer_na, LexContext, Lexer};
+use orchid_extension::lexer::{err_not_applicable, LexContext, Lexer};
 use orchid_extension::tree::{GenTok, GenTokTree};
 
 use super::str_atom::IntStrAtom;
@@ -96,33 +96,31 @@ pub struct StringLexer;
 impl Lexer for StringLexer {
   const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"'];
   fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
-    let mut tail = all.strip_prefix('"').ok_or_else(err_lexer_na)?;
+    let mut tail = all.strip_prefix('"').ok_or_else(err_not_applicable)?;
-    let mut parts = Vec::<GenTokTree<'a>>::new();
+    let mut ret = GenTok::X(IntStrAtom::from(intern!(str: "")).factory()).at(ctx.tok_ran(0, all));
     let mut cur = String::new();
     let mut errors = vec![];
-    let commit_str =
+    let str_to_gen = |str: &mut String, tail: &str, err: &mut Vec<OrcErr>| {
-      |str: &mut String, tail: &str, err: &mut Vec<OrcErr>, parts: &mut Vec<GenTokTree<'a>>| {
+      let str_val = parse_string(&str.split_off(0))
-        let str_val = parse_string(str)
         .inspect_err(|e| err.push(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
         .unwrap_or_default();
-        let tok = GenTok::X(IntStrAtom::from(intern(&*str_val)).factory());
+      GenTok::X(IntStrAtom::from(intern(&*str_val)).factory())
-        parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
+        .at(ctx.tok_ran(str.len() as u32, tail))
-        *str = String::new();
+    };
+    let add_frag = |prev: GenTokTree<'a>, new: GenTokTree<'a>| {
+      let range = prev.range.start..new.range.end;
+      wrap_tokv(vname_tv(&vname!(std::string::concat), new.range.end).chain([prev, new]), range)
     };
     loop {
       if let Some(rest) = tail.strip_prefix('"') {
-        commit_str(&mut cur, tail, &mut errors, &mut parts);
+        return Ok((rest, add_frag(ret, str_to_gen(&mut cur, tail, &mut errors))));
-        return Ok((rest, wrap_tokv(parts, ctx.pos(all)..ctx.pos(rest))));
       } else if let Some(rest) = tail.strip_prefix('$') {
-        commit_str(&mut cur, tail, &mut errors, &mut parts);
+        ret = add_frag(ret, str_to_gen(&mut cur, tail, &mut errors));
-        parts.push(GenTok::Name(intern!(str: "++")).at(ctx.tok_ran(1, rest)));
-        parts.extend(vname_tv(&vname!(std::string::convert), ctx.tok_ran(1, rest)));
         let (new_tail, tree) = ctx.recurse(rest)?;
         tail = new_tail;
-        parts.push(tree);
+        ret = add_frag(ret, tree);
       } else if tail.starts_with('\\') {
-        // parse_string will deal with it, we just have to make sure we skip the next
+        // parse_string will deal with it, we just have to skip the next char
-        // char
         tail = &tail[2..];
       } else {
         let mut ch = tail.chars();
@@ -131,12 +129,9 @@ impl Lexer for StringLexer {
           tail = ch.as_str();
         } else {
           let range = ctx.pos(all)..ctx.pos("");
-          commit_str(&mut cur, tail, &mut errors, &mut parts);
+          return Err(mk_errv(intern!(str: "No string end"), "String never terminated with \"", [
-          return Err(vec![mk_err(
+            Pos::Range(range.clone()).into(),
-            intern!(str: "No string end"),
+          ]));
-            "String never terminated with \"",
-            [Pos::Range(range.clone()).into()],
-          )]);
         }
       }
     }

xtask/src/main.rs

@@ -27,7 +27,7 @@ fn main() -> io::Result<ExitCode> {
   let args = Args::parse();
   match args.command {
     Commands::CheckApiRefs => walk_wsp(&mut |_| Ok(true), &mut |file| {
-      if file.path().extension() == Some(OsStr::new("rs")) || file.file_name() == "lib.rs" {
+      if file.path().extension() == Some(OsStr::new("rs")) && file.file_name() != "lib.rs" {
         let mut contents = String::new();
         File::open(file.path())?.read_to_string(&mut contents)?;
         for (l, line) in contents.lines().enumerate() {