Finally figured out how I want atoms to work

.vscode/settings.json

@@ -1,2 +1,4 @@
 {
+    "rust-analyzer.check.command": "check",
+    "rust-analyzer.rustfmt.extraArgs": ["+nightly"]
 }

Cargo.lock

@@ -259,6 +259,7 @@ dependencies = [
 name = "orchid-api-traits"
 version = "0.1.0"
 dependencies = [
+ "never",
  "ordered-float",
 ]
 
@@ -289,10 +290,14 @@ dependencies = [
  "derive_destructure",
  "hashbrown",
  "itertools",
+ "never",
  "orchid-api",
  "orchid-api-traits",
  "orchid-base",
  "ordered-float",
+ "substack",
+ "trait-set",
+ "typeid",
 ]
 
 [[package]]
@@ -491,6 +496,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "typeid"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf"
+
 [[package]]
 name = "typenum"
 version = "1.17.0"

orchid-api-derive/src/decode.rs

@@ -12,7 +12,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
   let decode = decode_body(&input.data);
   let expanded = quote! {
     impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause {
-      fn decode<R: std::io::Read>(read: &mut R) -> Self { #decode }
+      fn decode<R: std::io::Read + ?Sized>(read: &mut R) -> Self { #decode }
     }
   };
   TokenStream::from(expanded)

orchid-api-derive/src/encode.rs

@@ -1,4 +1,3 @@
-
 use proc_macro::TokenStream;
 use proc_macro2 as pm2;
 use quote::ToTokens;
@@ -15,7 +14,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
   let encode = encode_body(&input.data);
   let expanded = quote! {
     impl #e_impl_generics orchid_api_traits::Encode for #name #e_ty_generics #e_where_clause {
-      fn encode<W: std::io::Write>(&self, write: &mut W) { #encode }
+      fn encode<W: std::io::Write + ?Sized>(&self, write: &mut W) { #encode }
     }
   };
   TokenStream::from(expanded)
@@ -65,4 +64,3 @@ fn encode_items(fields: &syn::Fields) -> Option<pm2::TokenStream> {
       Some(encode_names((0..fields.len()).map(|i| pos_field_name(i, un.span())))),
   }
 }
-

orchid-api-derive/src/hierarchy.rs

@@ -60,7 +60,7 @@ fn gen_casts(ancestry: &[pm2::TokenStream], this: &pm2::TokenStream) -> pm2::Tok
               _ => false
             }
           }
-        }
+        },
       }
     }
     let chk = gen_chk(inter, this);
@@ -75,7 +75,7 @@ fn gen_casts(ancestry: &[pm2::TokenStream], this: &pm2::TokenStream) -> pm2::Tok
               _ => unreachable!("Checked above!"),
             }
           }
-        }
+        },
       }
     }
     let unpk = gen_unpk(inter, this);
@@ -104,8 +104,8 @@ fn get_ancestry(input: &DeriveInput) -> Option<Vec<pm2::TokenStream>> {
     match &attr.meta {
       syn::Meta::List(list) => (list.tokens.clone().into_iter())
         .batching(|it| {
-          let grp: pm2::TokenStream = it
+          let grp: pm2::TokenStream =
-            .take_while(|t| {
+            it.take_while(|t| {
               if let TokenTree::Punct(punct) = t { punct.as_char() != ',' } else { true }
             })
             .collect();
@@ -122,6 +122,4 @@ fn is_extendable(input: &DeriveInput) -> bool {
 }
 
 #[test]
-fn test_wtf() {
+fn test_wtf() { eprintln!("{}", gen_casts(&[quote!(ExtHostReq)], &quote!(BogusReq))) }
-  eprintln!("{}", gen_casts(&[quote!(ExtHostReq)], &quote!(BogusReq)))
-}

orchid-api-traits/Cargo.toml

@@ -6,4 +6,5 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+never = "0.1.0"
 ordered-float = "4.2"

orchid-api-traits/src/coding.rs

@@ -6,6 +6,7 @@ use std::ops::{Range, RangeInclusive};
 use std::rc::Rc;
 use std::sync::Arc;
 
+use never::Never;
 use ordered_float::{FloatCore, NotNan};
 
 use crate::encode_enum;
@@ -13,31 +14,35 @@ use crate::encode_enum;
 pub trait Decode {
   /// Decode an instance from the beginning of the buffer. Return the decoded
   /// data and the remaining buffer.
-  fn decode<R: Read>(read: &mut R) -> Self;
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self;
 }
 pub trait Encode {
   /// Append an instance of the struct to the buffer
-  fn encode<W: Write>(&self, write: &mut W);
+  fn encode<W: Write + ?Sized>(&self, write: &mut W);
   fn enc_vec(&self) -> Vec<u8> {
     let mut vec = Vec::new();
     self.encode(&mut vec);
     vec
   }
 }
-pub trait Coding: Encode + Decode + Clone {}
+pub trait Coding: Encode + Decode + Clone {
+  fn get_decoder<T>(map: impl Fn(Self) -> T + 'static) -> impl Fn(&mut dyn Read) -> T {
+    move |r| map(Self::decode(r))
+  }
+}
 impl<T: Encode + Decode + Clone> Coding for T {}
 
 macro_rules! num_impl {
   ($number:ty, $size:expr) => {
     impl Decode for $number {
-      fn decode<R: Read>(read: &mut R) -> Self {
+      fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
         let mut bytes = [0u8; $size];
         read.read_exact(&mut bytes).unwrap();
         <$number>::from_be_bytes(bytes)
       }
     }
     impl Encode for $number {
-      fn encode<W: Write>(&self, write: &mut W) {
+      fn encode<W: Write + ?Sized>(&self, write: &mut W) {
         write.write_all(&self.to_be_bytes()).expect("Could not write number")
       }
     }
@@ -62,10 +67,10 @@ num_impl!(f32, 4);
 macro_rules! nonzero_impl {
   ($name:ty) => {
     impl Decode for $name {
-      fn decode<R: Read>(read: &mut R) -> Self { Self::new(Decode::decode(read)).unwrap() }
+      fn decode<R: Read + ?Sized>(read: &mut R) -> Self { Self::new(Decode::decode(read)).unwrap() }
     }
     impl Encode for $name {
-      fn encode<W: Write>(&self, write: &mut W) { self.get().encode(write) }
+      fn encode<W: Write + ?Sized>(&self, write: &mut W) { self.get().encode(write) }
     }
   };
 }
@@ -82,16 +87,18 @@ nonzero_impl!(std::num::NonZeroI64);
 nonzero_impl!(std::num::NonZeroI128);
 
 impl<'a, T: Encode + 'a> Encode for &'a T {
-  fn encode<W: Write>(&self, write: &mut W) { (**self).encode(write) }
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
 }
 impl<T: Decode + FloatCore> Decode for NotNan<T> {
-  fn decode<R: Read>(read: &mut R) -> Self { NotNan::new(T::decode(read)).expect("Float was NaN") }
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+    NotNan::new(T::decode(read)).expect("Float was NaN")
+  }
 }
 impl<T: Encode + FloatCore> Encode for NotNan<T> {
-  fn encode<W: Write>(&self, write: &mut W) { self.as_ref().encode(write) }
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) { self.as_ref().encode(write) }
 }
 impl Decode for String {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     let len = u64::decode(read).try_into().unwrap();
     let mut data = vec![0u8; len];
     read.read_exact(&mut data).unwrap();
@@ -99,37 +106,37 @@ impl Decode for String {
   }
 }
 impl Encode for String {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     u64::try_from(self.len()).unwrap().encode(write);
     write.write_all(self.as_bytes()).unwrap()
   }
 }
 impl Encode for str {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     u64::try_from(self.len()).unwrap().encode(write);
     write.write_all(self.as_bytes()).unwrap()
   }
 }
 impl<T: Decode> Decode for Vec<T> {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     let len = u64::decode(read).try_into().unwrap();
     iter::repeat_with(|| T::decode(read)).take(len).collect()
   }
 }
 impl<T: Encode> Encode for Vec<T> {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     u64::try_from(self.len()).unwrap().encode(write);
     self.iter().for_each(|t| t.encode(write));
   }
 }
 impl<T: Encode> Encode for [T] {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     u64::try_from(self.len()).unwrap().encode(write);
     self.iter().for_each(|t| t.encode(write));
   }
 }
 impl<T: Decode> Decode for Option<T> {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     match u8::decode(read) {
       0 => None,
       1 => Some(T::decode(read)),
@@ -138,14 +145,14 @@ impl<T: Decode> Decode for Option<T> {
   }
 }
 impl<T: Encode> Encode for Option<T> {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     let t = if let Some(t) = self { t } else { return 0u8.encode(write) };
     1u8.encode(write);
     t.encode(write);
   }
 }
 impl<T: Decode, E: Decode> Decode for Result<T, E> {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     match u8::decode(read) {
       0 => Self::Ok(T::decode(read)),
       1 => Self::Err(E::decode(read)),
@@ -155,7 +162,7 @@ impl<T: Decode, E: Decode> Decode for Result<T, E> {
 }
 
 impl<T: Encode, E: Encode> Encode for Result<T, E> {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     match self {
       Ok(t) => encode_enum(write, 0, |w| t.encode(w)),
       Err(e) => encode_enum(write, 1, |w| e.encode(w)),
@@ -163,13 +170,13 @@ impl<T: Encode, E: Encode> Encode for Result<T, E> {
   }
 }
 impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     let len = u64::decode(read).try_into().unwrap();
     iter::repeat_with(|| <(K, V)>::decode(read)).take(len).collect()
   }
 }
 impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     u64::try_from(self.len()).unwrap().encode(write);
     self.iter().for_each(|pair| pair.encode(write));
   }
@@ -177,10 +184,10 @@ impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
 macro_rules! tuple {
   (($($t:ident)*) ($($T:ident)*)) => {
     impl<$($T: Decode),*> Decode for ($($T,)*) {
-      fn decode<R: Read>(read: &mut R) -> Self { ($($T::decode(read),)*) }
+      fn decode<R: Read + ?Sized>(read: &mut R) -> Self { ($($T::decode(read),)*) }
     }
     impl<$($T: Encode),*> Encode for ($($T,)*) {
-      fn encode<W: Write>(&self, write: &mut W) {
+      fn encode<W: Write + ?Sized>(&self, write: &mut W) {
         let ($($t,)*) = self;
         $( $t.encode(write); )*
       }
@@ -206,41 +213,49 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I));
 tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16
 
 impl Decode for () {
-  fn decode<R: Read>(_: &mut R) -> Self {}
+  fn decode<R: Read + ?Sized>(_: &mut R) -> Self {}
 }
 impl Encode for () {
-  fn encode<W: Write>(&self, _: &mut W) {}
+  fn encode<W: Write + ?Sized>(&self, _: &mut W) {}
+}
+impl Decode for Never {
+  fn decode<R: Read + ?Sized>(_: &mut R) -> Self {
+    unreachable!("A value of Never cannot exist so it can't have been serialized");
+  }
+}
+impl Encode for Never {
+  fn encode<W: Write + ?Sized>(&self, _: &mut W) { match *self {} }
 }
 impl Decode for bool {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     let mut buf = [0];
     read.read_exact(&mut buf).unwrap();
     buf[0] != 0
   }
 }
 impl Encode for bool {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     write.write_all(&[if *self { 0xff } else { 0 }]).unwrap()
   }
 }
 impl<T: Decode, const N: usize> Decode for [T; N] {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     // TODO: figure out how to do this in safe rust on the stack
     ((0..N).map(|_| T::decode(read)).collect::<Vec<_>>().try_into())
       .unwrap_or_else(|_| unreachable!("The length of this iterator is statically known"))
   }
 }
 impl<T: Encode, const N: usize> Encode for [T; N] {
-  fn encode<W: Write>(&self, write: &mut W) { self.iter().for_each(|t| t.encode(write)) }
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) { self.iter().for_each(|t| t.encode(write)) }
 }
 
 macro_rules! two_end_range {
   ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => {
     impl<T: Decode> Decode for $name<T> {
-      fn decode<R: Read>(read: &mut R) -> Self { T::decode(read) $op T::decode(read) }
+      fn decode<R: Read + ?Sized>(read: &mut R) -> Self { T::decode(read) $op T::decode(read) }
     }
     impl<T: Encode> Encode for $name<T> {
-      fn encode<W: Write>(&self, write: &mut W) {
+      fn encode<W: Write + ?Sized>(&self, write: &mut W) {
         let $this = self;
         ($start).encode(write);
         ($end).encode(write);
@@ -255,10 +270,10 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end());
 macro_rules! smart_ptr {
   ($name:tt) => {
     impl<T: Decode> Decode for $name<T> {
-      fn decode<R: Read>(read: &mut R) -> Self { $name::new(T::decode(read)) }
+      fn decode<R: Read + ?Sized>(read: &mut R) -> Self { $name::new(T::decode(read)) }
     }
     impl<T: Encode> Encode for $name<T> {
-      fn encode<W: Write>(&self, write: &mut W) { (**self).encode(write) }
+      fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
     }
   };
 }
@@ -268,8 +283,8 @@ smart_ptr!(Rc);
 smart_ptr!(Box);
 
 impl Decode for char {
-  fn decode<R: Read>(read: &mut R) -> Self { char::from_u32(u32::decode(read)).unwrap() }
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self { char::from_u32(u32::decode(read)).unwrap() }
 }
 impl Encode for char {
-  fn encode<W: Write>(&self, write: &mut W) { (*self as u32).encode(write) }
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) { (*self as u32).encode(write) }
 }

orchid-api-traits/src/helpers.rs

@@ -2,16 +2,16 @@ use std::io::{Read, Write};
 
 use crate::Encode;
 
-pub fn encode_enum<W: Write>(write: &mut W, id: u8, f: impl FnOnce(&mut W)) {
+pub fn encode_enum<W: Write + ?Sized>(write: &mut W, id: u8, f: impl FnOnce(&mut W)) {
   id.encode(write);
   f(write)
 }
 
-pub fn write_exact(write: &mut impl Write, bytes: &'static [u8]) {
+pub fn write_exact<W: Write + ?Sized>(write: &mut W, bytes: &'static [u8]) {
   write.write_all(bytes).expect("Failed to write exact bytes")
 }
 
-pub fn read_exact(read: &mut impl Read, bytes: &'static [u8]) {
+pub fn read_exact<R: Read + ?Sized>(read: &mut R, bytes: &'static [u8]) {
   let mut data = vec![0u8; bytes.len()];
   read.read_exact(&mut data).expect("Failed to read bytes");
   assert_eq!(&data, bytes, "Wrong bytes")

orchid-api/src/atom.rs

@@ -7,6 +7,14 @@ use crate::system::SysId;
 
 pub type AtomData = Vec<u8>;
 
+/// An atom owned by an implied system. Usually used in responses from a system.
+/// This has the same semantics as [Atom] except in that the owner is implied.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct LocalAtom {
+  pub drop: bool,
+  pub data: AtomData,
+}
+
 /// An atom representation that can be serialized and sent around. Atoms
 /// represent the smallest increment of work.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]

orchid-api/src/error.rs

@@ -1 +1,35 @@
+use orchid_api_derive::Coding;
+
+use crate::intern::TStr;
+use crate::location::Location;
+
 pub type ProjErrId = u16;
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct ProjErrLocation {
+  /// Description of the relation of this location to the error. If not used,
+  /// set to empty string
+  message: String,
+  /// Location in code where the error emerged. This is usually [Location::Gen].
+  location: Location,
+}
+
+/// Programming errors raised by extensions. At runtime these produce the
+/// equivalent of a Haskell bottom. Note that runtime errors produced by
+/// fallible operations should return an Orchid result and not a bottom.
+/// For example, file reading produces result::err when the file doesn't exist,
+/// and a bottom if the file name isn't a string.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct ProjErr {
+  /// General description of the kind of error.
+  description: TStr,
+  /// Specific information about the exact error, preferably containing concrete
+  /// values.
+  message: String,
+  /// Specific code fragments that have contributed to the emergence of the
+  /// error.
+  locations: Vec<ProjErrLocation>,
+}
+
+/// If this is an [`Err`] then the [`Vec`] must not be empty.
+pub type ProjResult<T> = Result<T, Vec<ProjErr>>;

orchid-api/src/expr.rs

@@ -1,7 +1,7 @@
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::atom::Atom;
+use crate::atom::LocalAtom;
 use crate::intern::{TStr, TStrv};
 use crate::location::Location;
 use crate::proto::{ExtHostNotif, ExtHostReq};
@@ -67,9 +67,13 @@ pub enum Clause {
   /// The lhs must be fully processed before the rhs can be processed.
   /// Equivalent to Haskell's function of the same name
   Seq(Box<Expr>, Box<Expr>),
-  /// Insert an atom in the tree. When the clause is used in the const tree, the
+  /// Insert a new atom in the tree. When the clause is used in the const tree,
-  /// atom must be trivial.
+  /// the atom must be trivial. This is always a newly constructed atom, if you
-  Atom(Atom),
+  /// want to reference an existing atom, use the corresponding [ExprTicket].
+  /// Because the atom is newly constructed, it also must belong to this system.
+  /// For convenience, [SysId::MAX] is also accepted as referring to this
+  /// system.
+  Atom(LocalAtom),
   /// A reference to a constant
   Const(TStrv),
 }
@@ -77,7 +81,7 @@ pub enum Clause {
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub struct Expr {
   pub clause: Clause,
-  pub location: Location
+  pub location: Location,
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]

orchid-api/src/fs.rs

@@ -1,16 +0,0 @@
-use orchid_api_derive::Coding;
-use orchid_api_traits::Request;
-
-pub type FsId = u16;
-
-#[derive(Clone, Debug, Coding)]
-pub enum Loaded {
-  Code(String),
-  Collection(Vec<String>),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct FsRead(pub Vec<String>);
-impl Request for FsRead {
-  type Response = Result<Loaded, ()>;
-}

orchid-api/src/lib.rs

@@ -1,10 +1,10 @@
 pub mod atom;
 pub mod error;
 pub mod expr;
-pub mod fs;
 pub mod intern;
 pub mod location;
+pub mod parser;
 pub mod proto;
 pub mod system;
 pub mod tree;
-pub mod parser;
+pub mod vfs;

orchid-api/src/location.rs

@@ -2,11 +2,14 @@ use std::ops::Range;
 
 use orchid_api_derive::Coding;
 
-use crate::intern::TStrv;
+use crate::intern::{TStr, TStrv};
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub enum Location {
   None,
+  /// Used in functions to denote the generated code that carries on the
+  /// location of the call. Not allowed in the const tree.
+  Inherit,
   Gen(CodeGenInfo),
   Range(SourceRange),
 }
@@ -19,6 +22,6 @@ pub struct SourceRange {
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub struct CodeGenInfo {
-  pub generator: String,
+  pub generator: TStr,
-  pub details: String,
+  pub details: TStr,
 }

orchid-api/src/parser.rs

@@ -3,63 +3,50 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
+use crate::error::ProjResult;
 use crate::intern::TStr;
 use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
 use crate::tree::TokenTree;
 
+/// - All ranges contain at least one character
+/// - All ranges are in increasing characeter order
+/// - There are excluded characters between each pair of neighboring ranges
+#[derive(Clone, Debug, Coding)]
+pub struct CharFilter(pub Vec<RangeInclusive<char>>);
+
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 #[extendable]
 pub enum ParserReq {
-  MkLexer(MkLexer),
   Lex(Lex),
 }
 
-pub type LexerId = u16;
-
-#[derive(Clone, Debug, Coding)]
-pub struct Lexer {
-  id: LexerId,
-  drop: bool,
-  notify_chars: Option<Vec<RangeInclusive<char>>>,
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ParserReq, HostExtReq)]
-pub struct MkLexer(pub SysId, pub TStr);
-impl Request for MkLexer {
-  type Response = Lexer;
-}
-
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ParserReq, HostExtReq)]
 pub struct Lex {
-  pub parser: LexerId,
+  pub sys: SysId,
-  pub next: char,
+  pub text: TStr,
   pub pos: u32,
 }
 impl Request for Lex {
-  type Response = Option<LexResult>;
+  type Response = Option<ProjResult<Lexed>>;
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct LexResult {
+pub struct Lexed {
-  pub consumed: u32,
+  pub pos: u32,
   pub data: TokenTree,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct SubLex {
-  pub lexer: LexerId,
+  pub text: TStr,
   pub pos: u32,
 }
 impl Request for SubLex {
-  type Response = SubLex;
+  type Response = ProjResult<Lexed>;
 }
 
-#[derive(Clone, Debug, Coding)]
-pub struct LexerDrop;
-
 pub struct ParseLine {}

orchid-api/src/proto.rs

@@ -27,18 +27,18 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::{atom, expr, intern, parser, system, tree};
+use crate::{atom, expr, intern, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader;
 impl Decode for HostHeader {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, HOST_INTRO);
     Self
   }
 }
 impl Encode for HostHeader {
-  fn encode<W: Write>(&self, write: &mut W) { write_exact(write, HOST_INTRO) }
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) { write_exact(write, HOST_INTRO) }
 }
 
 static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
@@ -46,13 +46,13 @@ pub struct ExtensionHeader {
   pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, EXT_INTRO);
     Self { systems: Vec::decode(read) }
   }
 }
 impl Encode for ExtensionHeader {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     write_exact(write, EXT_INTRO);
     self.systems.encode(write)
   }
@@ -76,8 +76,8 @@ pub enum ExtHostReq {
 }
 
 /// Notifications sent from the extension to the host
-#[derive(Debug, Clone, Coding, Hierarchy)]
 #[allow(clippy::enum_variant_names)]
+#[derive(Debug, Clone, Coding, Hierarchy)]
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
@@ -99,6 +99,7 @@ pub enum HostExtReq {
   AtomReq(atom::AtomReq),
   ParserReq(parser::ParserReq),
   GetConstTree(tree::GetConstTree),
+  VfsReq(vfs::VfsReq),
 }
 
 /// Notifications sent from the host to the extension
@@ -107,7 +108,6 @@ pub enum HostExtReq {
 pub enum HostExtNotif {
   SystemDrop(system::SystemDrop),
   AtomDrop(atom::AtomDrop),
-  LexerDrop(parser::LexerDrop),
   /// The host can assume that after this notif is sent, a correctly written
   /// extension will eventually exit.
   Exit,

orchid-api/src/system.rs

@@ -2,6 +2,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
+use crate::parser::CharFilter;
 use crate::proto::{HostExtNotif, HostExtReq};
 
 /// ID of a system type
@@ -46,7 +47,15 @@ pub struct NewSystem {
   pub depends: Vec<SysId>,
 }
 impl Request for NewSystem {
-  type Response = ();
+  type Response = SystemInst;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct SystemInst {
+  /// The set of possible starting characters of tokens the lexer of this system
+  /// can process. The lexer will notify this system if it encounters one of
+  /// these characters.9
+  pub lex_filter: CharFilter,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/vfs.rs

@@ -0,0 +1,45 @@
+use std::collections::HashMap;
+
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::error::ProjResult;
+use crate::intern::TStr;
+use crate::proto::HostExtReq;
+use crate::system::SysId;
+
+pub type VfsId = u16;
+
+#[derive(Clone, Debug, Coding)]
+pub enum Loaded {
+  Code(String),
+  Collection(Vec<TStr>),
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(VfsReq, HostExtReq)]
+pub struct VfsRead(pub SysId, pub VfsId, pub Vec<TStr>);
+impl Request for VfsRead {
+  type Response = ProjResult<Loaded>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum EagerVfs {
+  Lazy(VfsId),
+  Eager(HashMap<TStr, EagerVfs>),
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(VfsReq, HostExtReq)]
+pub struct GetVfs(pub SysId);
+impl Request for GetVfs {
+  type Response = EagerVfs;
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+#[extendable]
+pub enum VfsReq {
+  GetVfs(GetVfs),
+  VfsRead(VfsRead),
+}

orchid-base/src/api_utils.rs

@@ -0,0 +1 @@
+

orchid-base/src/char_filter.rs

@@ -0,0 +1,45 @@
+use std::ops::RangeInclusive;
+
+use itertools::Itertools;
+use orchid_api::parser::CharFilter;
+
+pub type CRange = RangeInclusive<char>;
+
+fn try_merge_char_ranges(left: CRange, right: CRange) -> Result<CRange, (CRange, CRange)> {
+  match *left.end() as u32 + 1 < *right.start() as u32 {
+    true => Err((left, right)),
+    false => Ok(*left.start()..=*right.end()),
+  }
+}
+
+/// Process the character ranges to make them adhere to the structural
+/// requirements of [CharFilter]
+pub fn mk_char_filter(items: impl IntoIterator<Item = CRange>) -> CharFilter {
+  CharFilter(
+    (items.into_iter())
+      .filter(|r| *r.start() as u32 + 1 < *r.end() as u32)
+      .sorted_by_key(|r| *r.start() as u32)
+      .coalesce(try_merge_char_ranges)
+      .collect_vec(),
+  )
+}
+
+/// Decide whether a char filter matches a character via binary search
+pub fn char_filter_match(cf: &CharFilter, c: char) -> bool {
+  match cf.0.binary_search_by_key(&c, |l| *l.end()) {
+    Ok(_) => true,                      // c is the end of a range
+    Err(i) if i == cf.0.len() => false, // all ranges end before c
+    Err(i) => cf.0[i].contains(&c),     // c between cf.0[i-1]?.end and cf.0[i].end, check [i]
+  }
+}
+
+/// Merge two char filters into a filter that matches if either of the
+/// constituents would match.
+pub fn char_filter_union(l: &CharFilter, r: &CharFilter) -> CharFilter {
+  CharFilter(
+    (l.0.iter().merge_by(&r.0, |l, r| l.start() <= r.start()))
+      .cloned()
+      .coalesce(try_merge_char_ranges)
+      .collect_vec(),
+  )
+}

orchid-base/src/id_store.rs

@@ -0,0 +1,53 @@
+use std::fmt::Debug;
+use std::num::NonZeroU64;
+use std::ops::{Deref, DerefMut};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Mutex, MutexGuard, OnceLock};
+
+use hashbrown::HashMap;
+
+pub struct IdStore<T> {
+  table: OnceLock<Mutex<HashMap<NonZeroU64, T>>>,
+  id: AtomicU64,
+}
+impl<T> IdStore<T> {
+  pub const fn new() -> Self { Self { table: OnceLock::new(), id: AtomicU64::new(1) } }
+  pub fn add<R>(&self, t: T) -> R
+  where
+    NonZeroU64: TryInto<R>,
+    <NonZeroU64 as TryInto<R>>::Error: Debug,
+  {
+    let tbl = self.table.get_or_init(Mutex::default);
+    let mut tbl_g = tbl.lock().unwrap();
+    let id64: NonZeroU64 = self.id.fetch_add(1, Ordering::Relaxed).try_into().unwrap();
+    let id: R = id64.try_into().expect("Keyspace exhausted");
+    assert!(tbl_g.insert(id64, t).is_none(), "atom ID wraparound");
+    id
+  }
+  pub fn get(&self, id: impl Into<NonZeroU64>) -> Option<IdRecord<'_, T>> {
+    let tbl = self.table.get_or_init(Mutex::default);
+    let tbl_g = tbl.lock().unwrap();
+    let id64 = id.into();
+    if tbl_g.contains_key(&id64) { Some(IdRecord(id64, tbl_g)) } else { None }
+  }
+}
+
+impl<T> Default for IdStore<T> {
+  fn default() -> Self { Self::new() }
+}
+
+pub struct IdRecord<'a, T>(NonZeroU64, MutexGuard<'a, HashMap<NonZeroU64, T>>);
+impl<'a, T> IdRecord<'a, T> {
+  pub fn remove(mut self) -> T { self.1.remove(&self.0).unwrap() }
+}
+impl<'a, T> Deref for IdRecord<'a, T> {
+  type Target = T;
+  fn deref(&self) -> &Self::Target {
+    self.1.get(&self.0).expect("Existence checked on construction")
+  }
+}
+impl<'a, T> DerefMut for IdRecord<'a, T> {
+  fn deref_mut(&mut self) -> &mut Self::Target {
+    self.1.get_mut(&self.0).expect("Existence checked on construction")
+  }
+}

orchid-base/src/lib.rs

@@ -1,16 +1,18 @@
 pub mod boxed_iter;
-pub mod msg;
 pub mod clone;
 pub mod combine;
 pub mod event;
+pub mod msg;
 // pub mod gen;
+pub mod api_utils;
+pub mod char_filter;
+pub mod id_store;
 pub mod intern;
+pub mod join;
 pub mod location;
 pub mod name;
 pub mod proj_error;
 pub mod reqnot;
+pub mod sequence;
 pub mod tree;
 pub mod virt_fs;
-pub mod join;
-pub mod sequence;
-pub mod api_utils;

orchid-base/src/proj_error.rs

@@ -9,8 +9,8 @@ use std::{fmt, process};
 use dyn_clone::{clone_box, DynClone};
 use itertools::Itertools;
 
-use crate::location::CodeOrigin;
 use crate::boxed_iter::{box_once, BoxedIter};
+use crate::location::CodeOrigin;
 #[allow(unused)] // for doc
 use crate::virt_fs::CodeNotFound;
 
@@ -220,7 +220,15 @@ impl Reporter {
   /// will always return true in the future.
   pub fn failing(&self) -> bool { !self.0.borrow().is_empty() }
   /// Report a fatal error
-  pub fn report(&self, error: ProjectErrorObj) { self.0.borrow_mut().push(error) }
+  pub fn report(&self, error: ProjectErrorObj) {
+    match error.as_any_ref().downcast_ref::<MultiError>() {
+      None => self.0.borrow_mut().push(error),
+      Some(me) =>
+        for err in me.0.iter() {
+          self.report(err.clone())
+        },
+    }
+  }
   /// Catch a fatal error, report it, and substitute the value
   pub fn fallback<T>(&self, res: ProjectResult<T>, cb: impl FnOnce(ProjectErrorObj) -> T) -> T {
     res.inspect_err(|e| self.report(e.clone())).unwrap_or_else(cb)
@@ -277,7 +285,11 @@ impl ProjectError for MultiError {
     self.0.iter().flat_map(|e| {
       e.positions().map(|pos| {
         let emsg = e.message();
-        let msg = if let Some(pmsg) = pos.message { format!("{emsg}: {pmsg}") } else { emsg };
+        let msg = match pos.message {
+          None => emsg,
+          Some(s) if s.is_empty() => emsg,
+          Some(pmsg) => format!("{emsg}: {pmsg}"),
+        };
         ErrorPosition { origin: pos.origin, message: Some(msg) }
       })
     })

orchid-base/src/reqnot.rs

@@ -1,3 +1,4 @@
+use std::marker::PhantomData;
 use std::mem;
 use std::ops::{BitAnd, Deref};
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -12,7 +13,8 @@ use trait_set::trait_set;
 trait_set! {
   pub trait SendFn<T: MsgSet> = for<'a> FnMut(&'a [u8], ReqNot<T>) + DynClone + Send + 'static;
   pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + Send + 'static;
-  pub trait NotifFn<T: MsgSet> = for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + Send + Sync + 'static;
+  pub trait NotifFn<T: MsgSet> =
+    for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + Send + Sync + 'static;
 }
 
 fn get_id(message: &[u8]) -> (u64, &[u8]) {
@@ -22,20 +24,24 @@ fn get_id(message: &[u8]) -> (u64, &[u8]) {
 pub struct RequestHandle<T: MsgSet> {
   id: u64,
   message: <T::In as Channel>::Req,
-  send: Box<dyn SendFn<T>>,
   parent: ReqNot<T>,
   fulfilled: AtomicBool,
 }
-impl<MS: MsgSet> RequestHandle<MS> {
+impl<MS: MsgSet + 'static> RequestHandle<MS> {
   pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
   pub fn req(&self) -> &<MS::In as Channel>::Req { &self.message }
   fn respond(&self, response: &impl Encode) {
     assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded");
     let mut buf = (!self.id).to_be_bytes().to_vec();
     response.encode(&mut buf);
-    clone_box(&*self.send)(&buf, self.parent.clone());
+    let mut send = clone_box(&*self.reqnot().0.lock().unwrap().send);
+    (send)(&buf, self.parent.clone());
   }
   pub fn handle<T: Request>(&self, _: &T, rep: &T::Response) { self.respond(rep) }
+  pub fn will_handle_as<T: Request>(&self, _: &T) -> ReqTypToken<T> { ReqTypToken(PhantomData) }
+  pub fn handle_as<T: Request>(&self, token: ReqTypToken<T>, rep: &T::Response) {
+    self.respond(rep)
+  }
 }
 impl<MS: MsgSet> Drop for RequestHandle<MS> {
   fn drop(&mut self) {
@@ -43,6 +49,8 @@ impl<MS: MsgSet> Drop for RequestHandle<MS> {
   }
 }
 
+pub struct ReqTypToken<T>(PhantomData<T>);
+
 pub fn respond_with<R: Request>(r: &R, f: impl FnOnce(&R) -> R::Response) -> Vec<u8> {
   r.respond(f(r))
 }
@@ -83,9 +91,8 @@ impl<T: MsgSet> ReqNot<T> {
       let sender = g.responses.remove(&!id).expect("Received response for invalid message");
       sender.send(message).unwrap();
     } else {
-      let send = clone_box(&*g.send);
       let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
-      (g.req)(RequestHandle { id, message, send, fulfilled: false.into(), parent: self.clone() })
+      (g.req)(RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() })
     }
   }
 
@@ -159,7 +166,8 @@ mod test {
   use orchid_api_traits::{Channel, Request};
 
   use super::{MsgSet, ReqNot};
-  use crate::{clone, reqnot::Requester as _};
+  use crate::clone;
+  use crate::reqnot::Requester as _;
 
   #[derive(Clone, Debug, Coding, PartialEq)]
   pub struct TestReq(u8);

orchid-base/src/virt_fs/common.rs

@@ -1,9 +1,9 @@
 use std::rc::Rc;
 use std::sync::Arc;
 
-use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
 use crate::intern::Token;
 use crate::name::{PathSlice, VPath};
+use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
 
 /// Represents the result of loading code from a string-tree form such
 /// as the file system. Cheap to clone.

orchid-base/src/virt_fs/decl.rs

@@ -3,11 +3,11 @@ use std::sync::Arc;
 
 use super::common::CodeNotFound;
 use super::{FSResult, Loaded, VirtFS};
-use crate::intern::Token;
-use crate::proj_error::ErrorSansOrigin;
-use crate::name::PathSlice;
-use crate::tree::{ModEntry, ModMember};
 use crate::combine::Combine;
+use crate::intern::Token;
+use crate::name::PathSlice;
+use crate::proj_error::ErrorSansOrigin;
+use crate::tree::{ModEntry, ModMember};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct ConflictingTrees;

orchid-base/src/virt_fs/dir.rs

@@ -10,8 +10,8 @@ use hashbrown::HashMap;
 use super::common::CodeNotFound;
 use super::{FSResult, Loaded, VirtFS};
 use crate::intern::{intern, Token};
-use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
 use crate::name::PathSlice;
+use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
 
 #[derive(Clone)]
 struct OpenError {

orchid-base/src/virt_fs/embed.rs

@@ -6,9 +6,9 @@ use rust_embed::RustEmbed;
 use super::common::CodeNotFound;
 use super::{FSResult, Loaded, VirtFS};
 use crate::intern::{intern, Token};
-use crate::proj_error::ErrorSansOrigin;
 use crate::location::CodeGenInfo;
 use crate::name::PathSlice;
+use crate::proj_error::ErrorSansOrigin;
 use crate::tree::{ModEntry, ModMember, Module};
 
 /// An in-memory FS tree for libraries managed internally by the interpreter

orchid-extension/Cargo.toml

@@ -10,7 +10,11 @@ ahash = "0.8.11"
 derive_destructure = "1.0.0"
 hashbrown = "0.14.5"
 itertools = "0.12.1"
+never = "0.1.0"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "4.2.0"
+substack = "1.1.0"
+trait-set = "0.3.0"
+typeid = "1.0.0"

orchid-extension/src/atom.rs

@@ -0,0 +1,174 @@
+use std::any::{type_name, Any};
+use std::io::{Read, Write};
+use std::num::NonZeroU64;
+use std::ops::Deref;
+use std::sync::Arc;
+use std::{fmt, mem};
+
+use derive_destructure::destructure;
+use orchid_api::atom::{Atom, Fwd};
+use orchid_api::expr::{ExprTicket, Release};
+use orchid_api::system::SysId;
+use orchid_api_traits::{Coding, Decode, Encode, Request};
+use orchid_base::id_store::{IdRecord, IdStore};
+use orchid_base::reqnot::Requester as _;
+use typeid::ConstTypeId;
+
+use crate::expr::GenClause;
+use crate::other_system::SystemHandle;
+use crate::system::{DynSystemCard, SystemCard};
+
+pub trait AtomCard: 'static + Sized {
+  type Owner: SystemCard;
+  type Data: Clone + Coding + Sized;
+  type Req: Coding;
+}
+
+pub fn get_info<A: AtomCard>(sys: &(impl DynSystemCard + ?Sized)) -> (usize, &AtomInfo) {
+  sys.atom_info_for(ConstTypeId::of::<A>()).unwrap_or_else(|| {
+    panic!("Atom {} not associated with system {}", type_name::<A>(), sys.name())
+  })
+}
+
+pub fn encode_atom_nodrop<A: AtomCard>(
+  sys_id: SysId,
+  sys: &(impl DynSystemCard + ?Sized),
+  data: &A::Data,
+) -> Atom {
+  let (info_pos, _) = get_info::<A>(sys);
+  let mut buf = (info_pos as u64).enc_vec();
+  data.encode(&mut buf);
+  Atom { owner: sys_id, drop: false, data: buf }
+}
+
+pub fn encode_atom_drop<A: AtomCard>(
+  sys_id: SysId,
+  sys: &(impl DynSystemCard + ?Sized),
+  atom_id: u64,
+  data: &A::Data,
+) -> Atom {
+  let (info_pos, _) = get_info::<A>(sys);
+  let mut buf = (info_pos as u64).enc_vec();
+  atom_id.encode(&mut buf);
+  data.encode(&mut buf);
+  Atom { owner: sys_id, drop: true, data: buf }
+}
+
+pub fn decode_atom<A: AtomCard>(
+  sys: &(impl DynSystemCard + ?Sized),
+  Atom { data, drop, owner: _ }: &Atom,
+) -> Option<A::Data> {
+  let (info_pos, info) = get_info::<A>(sys);
+  let mut data = &data[..];
+  if u64::decode(&mut data) != info_pos as u64 {
+    return None;
+  }
+  let val = (info.decode)(data);
+  Some(*val.downcast().expect("The type-id checked out, the decode should've worked"))
+}
+
+#[derive(destructure)]
+pub struct ForeignAtom<A: AtomCard> {
+  pub(crate) sys: SystemHandle<A::Owner>,
+  pub(crate) ticket: ExprTicket,
+  pub(crate) api: Atom,
+  pub(crate) value: A::Data,
+}
+impl<A: AtomCard> ForeignAtom<A> {
+  /// Unpack the object, returning the held atom and expr ticket. This is in
+  /// contrast to dropping the atom which releases the expr ticket.
+  pub fn unpack(self) -> (A::Data, ExprTicket, Atom) {
+    let (_, ticket, api, value) = self.destructure();
+    (value, ticket, api)
+  }
+  pub fn ticket(&self) -> ExprTicket { self.ticket }
+  pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
+    R::Response::decode(&mut &self.sys.reqnot.request(Fwd(self.api.clone(), req.enc_vec()))[..])
+  }
+}
+impl<A: AtomCard> Deref for ForeignAtom<A> {
+  type Target = A::Data;
+  fn deref(&self) -> &Self::Target { &self.value }
+}
+impl<A: AtomCard> Drop for ForeignAtom<A> {
+  fn drop(&mut self) { self.sys.reqnot.notify(Release(self.sys.id(), self.ticket)) }
+}
+
+pub struct AtomInfo {
+  pub tid: ConstTypeId,
+  pub decode: fn(&[u8]) -> Box<dyn Any>,
+  pub call: fn(&[u8], ExprTicket) -> GenClause,
+  pub call_ref: fn(&[u8], ExprTicket) -> GenClause,
+  pub same: fn(&[u8], &[u8]) -> bool,
+  pub handle_req: fn(&[u8], &mut dyn Read, &mut dyn Write),
+  pub drop: fn(&[u8]),
+}
+
+pub trait ThinAtom: AtomCard<Data = Self> + Coding + fmt::Debug {
+  fn call(&self, arg: ExprTicket) -> GenClause;
+  fn same(&self, other: &Self) -> bool;
+  fn handle_req(&self, req: Self::Req, rep: &mut (impl Write + ?Sized));
+}
+
+pub const fn thin_atom_info<T: ThinAtom>() -> AtomInfo {
+  AtomInfo {
+    tid: ConstTypeId::of::<T>(),
+    decode: |mut b| Box::new(T::decode(&mut b)),
+    call: |mut b, extk| T::decode(&mut b).call(extk),
+    call_ref: |mut b, extk| T::decode(&mut b).call(extk),
+    handle_req: |mut b, req, rep| T::decode(&mut b).handle_req(Decode::decode(req), rep),
+    same: |mut b1, mut b2| T::decode(&mut b1).same(&T::decode(&mut b2)),
+    drop: |mut b1| eprintln!("Received drop signal for non-drop atom {:?}", T::decode(&mut b1)),
+  }
+}
+
+/// Atoms that have a [Drop]
+pub trait OwnedAtom: AtomCard + Deref<Target = Self::Data> + Send + Sync + Any + 'static {
+  fn call_ref(&self, arg: ExprTicket) -> GenClause;
+  fn call(self, arg: ExprTicket) -> GenClause;
+  fn same(&self, other: &Self) -> bool;
+  fn handle_req(&self, req: Self::Req, rep: &mut (impl Write + ?Sized));
+}
+
+pub trait DynOwnedAtom: Send + Sync + 'static {
+  fn atom_tid(&self) -> ConstTypeId;
+  fn as_any_ref(&self) -> &dyn Any;
+  fn dyn_call_ref(&self, arg: ExprTicket) -> GenClause;
+  fn dyn_call(self: Box<Self>, arg: ExprTicket) -> GenClause;
+  fn dyn_same(&self, other: &dyn DynOwnedAtom) -> bool;
+  fn dyn_handle_req(&self, req: &mut dyn Read, rep: &mut dyn Write);
+}
+
+impl<T: OwnedAtom> DynOwnedAtom for T {
+  fn atom_tid(&self) -> ConstTypeId { ConstTypeId::of::<T>() }
+  fn as_any_ref(&self) -> &dyn Any { self }
+  fn dyn_call_ref(&self, arg: ExprTicket) -> GenClause { self.call_ref(arg) }
+  fn dyn_call(self: Box<Self>, arg: ExprTicket) -> GenClause { self.call(arg) }
+  fn dyn_same(&self, other: &dyn DynOwnedAtom) -> bool {
+    if ConstTypeId::of::<Self>() != other.as_any_ref().type_id() {
+      return false;
+    }
+    let other_self = other.as_any_ref().downcast_ref().expect("The type_ids are the same");
+    self.same(other_self)
+  }
+  fn dyn_handle_req(&self, req: &mut dyn Read, rep: &mut dyn Write) {
+    self.handle_req(<Self as AtomCard>::Req::decode(req), rep)
+  }
+}
+
+pub(crate) static OBJ_STORE: IdStore<Box<dyn DynOwnedAtom>> = IdStore::new();
+
+const fn owned_atom_info<T: OwnedAtom>() -> AtomInfo {
+  fn with_atom<U>(mut b: &[u8], f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>) -> U) -> U {
+    f(OBJ_STORE.get(NonZeroU64::decode(&mut b)).expect("Received invalid atom ID"))
+  }
+  AtomInfo {
+    tid: ConstTypeId::of::<T>(),
+    decode: |mut b| Box::new(T::Data::decode(&mut b)),
+    call: |b, arg| with_atom(b, |a| a.remove().dyn_call(arg)),
+    call_ref: |b, arg| with_atom(b, |a| a.dyn_call_ref(arg)),
+    same: |b1, b2| with_atom(b1, |a1| with_atom(b2, |a2| a1.dyn_same(&**a2))),
+    handle_req: |b, req, rep| with_atom(b, |a| a.dyn_handle_req(req, rep)),
+    drop: |b| mem::drop(with_atom(b, |a| a.remove())),
+  }
+}

orchid-extension/src/data.rs

@@ -1,83 +0,0 @@
-use std::marker::PhantomData;
-use std::ops::Deref;
-
-use derive_destructure::destructure;
-use orchid_api::atom::{Atom, Fwd};
-use orchid_api::expr::{ExprTicket, Release};
-use orchid_api::proto::ExtMsgSet;
-use orchid_api::system::SysId;
-use orchid_api_traits::{Coding, Decode, Request};
-use orchid_base::reqnot::{ReqNot, Requester};
-
-pub struct SystemHandle<T: SystemDepCard> {
-  _t: PhantomData<T>,
-  id: SysId,
-  reqnot: ReqNot<ExtMsgSet>,
-}
-impl<T: SystemDepCard> SystemHandle<T> {
-  pub(crate) fn new(id: SysId, reqnot: ReqNot<ExtMsgSet>) -> Self {
-    Self { _t: PhantomData, id, reqnot }
-  }
-  pub fn id(&self) -> SysId { self.id }
-  pub fn wrap_atom<A: Atomic<Owner = T>>(
-    &self,
-    api: Atom,
-    ticket: ExprTicket,
-  ) -> Result<OwnedAtom<A>, Atom> {
-    if api.owner == self.id {
-      Ok(OwnedAtom { ticket, sys: self.clone(), value: T::decode_atom(&api), api })
-    } else {
-      Err(api)
-    }
-  }
-}
-impl<T: SystemDepCard> Clone for SystemHandle<T> {
-  fn clone(&self) -> Self { Self { reqnot: self.reqnot.clone(), _t: PhantomData, id: self.id } }
-}
-
-pub trait Atomic: 'static {
-  type Owner: SystemDepCard;
-  type Req: Coding;
-  const HAS_DROP: bool;
-}
-
-pub trait SystemDepCard: 'static {
-  const NAME: &'static str;
-  /// Decode an atom from binary representation.
-  ///
-  /// This is held in the dep card because there is no global type tag on the
-  /// atom, so by the logic of the binary coding algorithm the value has to be a
-  /// concrete type, probably an enum of the viable types.
-  fn decode_atom<A: Atomic<Owner = Self>>(api: &Atom) -> A;
-}
-
-#[derive(destructure)]
-pub struct OwnedAtom<A: Atomic> {
-  sys: SystemHandle<A::Owner>,
-  ticket: ExprTicket,
-  api: Atom,
-  value: A,
-}
-impl<A: Atomic> OwnedAtom<A> {
-  /// Unpack the object, returning the held atom and expr ticket. This is in
-  /// contrast to dropping the atom which releases the expr ticket.
-  pub fn unpack(self) -> (A, ExprTicket, Atom) {
-    let (_, ticket, api, value) = self.destructure();
-    (value, ticket, api)
-  }
-  pub fn ticket(&self) -> ExprTicket { self.ticket }
-  pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
-    R::Response::decode(&mut &self.sys.reqnot.request(Fwd(self.api.clone(), req.enc_vec()))[..])
-  }
-}
-impl<A: Atomic> Deref for OwnedAtom<A> {
-  type Target = A;
-  fn deref(&self) -> &Self::Target { &self.value }
-}
-impl<A: Atomic> Drop for OwnedAtom<A> {
-  fn drop(&mut self) {
-    if A::HAS_DROP {
-      self.sys.reqnot.notify(Release(self.sys.id(), self.ticket))
-    }
-  }
-}

orchid-extension/src/entrypoint.rs

@@ -1,34 +1,65 @@
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
+use std::{mem, thread};
 
 use hashbrown::HashMap;
 use itertools::Itertools;
+use orchid_api::atom::{Atom, AtomReq, AtomSame, CallRef, FinalCall, Fwded};
+use orchid_api::parser::{CharFilter, Lex, Lexed, ParserReq, SubLex};
 use orchid_api::proto::{ExtMsgSet, ExtensionHeader, HostExtNotif, HostExtReq, HostHeader};
+use orchid_api::system::{SysId, SystemInst};
+use orchid_api::vfs::{EagerVfs, VfsId, VfsRead, VfsReq};
 use orchid_api_traits::{Decode, Encode};
+use orchid_base::char_filter::{char_filter_union, mk_char_filter};
 use orchid_base::clone;
-use orchid_base::intern::{init_replica, sweep_replica};
+use orchid_base::intern::{deintern, init_replica, sweep_replica};
+use orchid_base::name::PathSlice;
 use orchid_base::reqnot::{ReqNot, Requester};
 
+use crate::atom::AtomInfo;
+use crate::fs::VirtFS;
 use crate::msg::{recv_parent_msg, send_parent_msg};
+use crate::system::DynSystem;
 use crate::system_ctor::DynSystemCtor;
 
 pub struct ExtensionData {
   pub systems: Vec<Box<dyn DynSystemCtor>>,
 }
 
+pub struct SystemRecord {
+  instance: Box<dyn DynSystem>,
+  vfses: HashMap<VfsId, Arc<dyn VirtFS>>,
+  declfs: EagerVfs,
+}
+
+pub fn with_atom_record<T>(
+  systems: &Mutex<HashMap<SysId, SystemRecord>>,
+  atom: &Atom,
+  cb: impl FnOnce(&AtomInfo, &[u8]) -> T,
+) -> T {
+  let mut data = &atom.data[..];
+  let systems_g = systems.lock().unwrap();
+  let sys = &systems_g[&atom.owner].instance;
+  let atom_record =
+    (sys.card().atoms()[u64::decode(&mut data) as usize].as_ref()).expect("Atom ID reserved");
+  cb(atom_record, data)
+}
+
 pub fn main(data: ExtensionData) {
   HostHeader::decode(&mut &recv_parent_msg().unwrap()[..]);
   let mut buf = Vec::new();
   let decls = data.systems.iter().map(|sys| sys.decl()).collect_vec();
-  let systems = Arc::new(Mutex::new(HashMap::new()));
+  let systems = Arc::new(Mutex::new(HashMap::<SysId, SystemRecord>::new()));
   ExtensionHeader { systems: decls.clone() }.encode(&mut buf);
   send_parent_msg(&buf).unwrap();
   let exiting = Arc::new(AtomicBool::new(false));
   let rn = ReqNot::<ExtMsgSet>::new(
     |a, _| send_parent_msg(a).unwrap(),
-    clone!(exiting; move |n, _| match n {
+    clone!(systems, exiting; move |n, _| match n {
       HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
-      _ => todo!(),
+      HostExtNotif::SystemDrop(sys) => mem::drop(systems.lock().unwrap().remove(&sys.0)),
+      HostExtNotif::AtomDrop(atom) =>
+        with_atom_record(&systems, &atom.0, |rec, data| (rec.drop)(data)),
     }),
     clone!(systems; move |req| match req.req() {
       HostExtReq::Ping(ping) => req.handle(ping, &()),
@@ -36,10 +67,57 @@ pub fn main(data: ExtensionData) {
       HostExtReq::NewSystem(new_sys) => {
         let i = decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system).unwrap().0;
         let system = data.systems[i].new_system(new_sys, req.reqnot());
-        systems.lock().unwrap().insert(new_sys.id, system);
+        let mut vfses = HashMap::new();
-        req.handle(new_sys, &())
+        let lex_filter = system.lexers().iter().fold(CharFilter(vec![]), |cf, lx| {
+          char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
+        });
+        systems.lock().unwrap().insert(new_sys.id, SystemRecord {
+          declfs: system.source().to_api_rec(&mut vfses),
+          vfses,
+          instance: system,
+        });
+        req.handle(new_sys, &SystemInst {
+          lex_filter
+        })
+      }
+      HostExtReq::GetConstTree(get_tree) => {
+        let systems_g = systems.lock().unwrap();
+        req.handle(get_tree, &systems_g[&get_tree.0].instance.env())
+      }
+      HostExtReq::VfsReq(VfsReq::GetVfs(get_vfs)) => {
+        let systems_g = systems.lock().unwrap();
+        req.handle(get_vfs, &systems_g[&get_vfs.0].declfs)
+      }
+      HostExtReq::VfsReq(VfsReq::VfsRead(vfs_read@VfsRead(sys_id, vfs_id, path))) => {
+        let systems_g = systems.lock().unwrap();
+        let path = path.iter().map(|t| deintern(*t)).collect_vec();
+        req.handle(vfs_read, &systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path)))
+      }
+      HostExtReq::ParserReq(ParserReq::Lex(lex)) => {
+        let systems_g = systems.lock().unwrap();
+        let Lex{ sys, text, pos } = *lex;
+        let lexers = systems_g[&sys].instance.lexers();
+        mem::drop(systems_g);
+        let source = deintern(text);
+        let tk = req.will_handle_as(lex);
+        thread::spawn(move || {
+          let reqnot = req.reqnot();
+          let mut recurse = |tail: &str| {
+            let pos = (source.len() - tail.len()) as u32;
+            let lexed = reqnot.request(SubLex{ pos, text })?;
+            Ok((&source[lexed.pos as usize..], lexed.data))
+          };
+          let lex_res = lexers.iter().find_map(|lx| lx.lex(&source[pos as usize..], &mut recurse));
+          req.handle_as(tk, &lex_res.map(|r| r.map(|(s, data)| {
+            let pos = (source.len() - s.len()) as u32;
+            Lexed { data, pos }
+          })))
+        });
       },
-      _ => todo!(),
+      HostExtReq::AtomReq(AtomReq::AtomSame(same@AtomSame(l, r))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(AtomReq::Fwded(call@Fwded(atom, req))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(AtomReq::CallRef(call@CallRef(atom, arg))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(AtomReq::FinalCall(call@FinalCall(atom, arg))) => todo!("subsys nimpl"),
     }),
   );
   init_replica(rn.clone().map());

orchid-extension/src/expr.rs

@@ -0,0 +1,53 @@
+use orchid_api::atom::Atom;
+use orchid_api::expr::ExprTicket;
+use orchid_api::system::SysId;
+use orchid_base::id_store::IdStore;
+use orchid_base::intern::Token;
+
+use crate::atom::{encode_atom_nodrop, DynOwnedAtom, OwnedAtom, ThinAtom, OBJ_STORE};
+use crate::system::DynSystem;
+
+pub enum GenClause {
+  Call(Box<GenClause>, Box<GenClause>),
+  Lambda(Token<String>, Box<GenClause>),
+  Arg(Token<String>),
+  Slot(ExprTicket),
+  Seq(Box<GenClause>, Box<GenClause>),
+  Const(Token<Vec<Token<String>>>),
+  ThinAtom(Box<dyn Fn(SysId, &dyn DynSystem) -> Atom>),
+  OwnedAtom(u64),
+}
+
+pub fn cnst(path: Token<Vec<Token<String>>>) -> GenClause { GenClause::Const(path) }
+pub fn val<A: ThinAtom>(atom: A) -> GenClause {
+  GenClause::ThinAtom(Box::new(move |id, sys| encode_atom_nodrop::<A>(id, sys.card(), &atom)))
+}
+
+pub fn obj<A: OwnedAtom>(atom: A) -> GenClause {
+  GenClause::OwnedAtom(OBJ_STORE.add(Box::new(atom)))
+}
+
+pub fn seq(ops: impl IntoIterator<Item = GenClause>) -> GenClause {
+  fn recur(mut ops: impl Iterator<Item = GenClause>) -> Option<GenClause> {
+    let op = ops.next()?;
+    Some(match recur(ops) {
+      None => op,
+      Some(rec) => GenClause::Seq(Box::new(op), Box::new(rec)),
+    })
+  }
+  recur(ops.into_iter()).expect("Empty list provided to seq!")
+}
+
+pub fn slot(extk: ExprTicket) -> GenClause { GenClause::Slot(extk) }
+
+pub fn arg(n: Token<String>) -> GenClause { GenClause::Arg(n) }
+
+pub fn lambda(n: Token<String>, b: impl IntoIterator<Item = GenClause>) -> GenClause {
+  GenClause::Lambda(n, Box::new(call(b)))
+}
+
+pub fn call(v: impl IntoIterator<Item = GenClause>) -> GenClause {
+  v.into_iter()
+    .reduce(|f, x| GenClause::Call(Box::new(f), Box::new(x)))
+    .expect("Empty call expression")
+}

orchid-extension/src/fs.rs

@@ -0,0 +1,48 @@
+use std::sync::Arc;
+
+use hashbrown::HashMap;
+use orchid_api::error::ProjResult;
+use orchid_api::vfs::{EagerVfs, Loaded, VfsId};
+use orchid_base::intern::{intern, Token};
+use orchid_base::name::PathSlice;
+use substack::Substack;
+use trait_set::trait_set;
+
+pub trait VirtFS: Send + Sync + 'static {
+  fn load(&self, path: &PathSlice) -> ProjResult<Loaded>;
+}
+
+trait_set! {
+  pub trait RecFsHandler<E> = FnMut(Substack<Token<String>>, &Arc<dyn VirtFS>) -> Result<(), E>;
+}
+
+pub enum DeclFs {
+  Lazy(Arc<dyn VirtFS>),
+  Mod(HashMap<Token<String>, DeclFs>),
+}
+impl DeclFs {
+  pub fn module(entries: impl IntoIterator<Item = (&'static str, Self)>) -> Self {
+    Self::Mod(entries.into_iter().map(|(k, v)| (intern(k), v)).collect())
+  }
+  fn rec<E>(&self, path: Substack<Token<String>>, f: &mut impl RecFsHandler<E>) -> Result<(), E> {
+    match self {
+      DeclFs::Lazy(fs) => f(path, fs),
+      DeclFs::Mod(entries) => entries.iter().try_for_each(|(k, v)| v.rec(path.push(k.clone()), f)),
+    }
+  }
+  pub fn recurse<E>(&self, f: &mut impl RecFsHandler<E>) -> Result<(), E> {
+    self.rec(Substack::Bottom, f)
+  }
+  pub fn to_api_rec(&self, vfses: &mut HashMap<VfsId, Arc<dyn VirtFS>>) -> EagerVfs {
+    match self {
+      DeclFs::Lazy(fs) => {
+        let id = vfses.len() as VfsId;
+        vfses.insert(id, fs.clone());
+        EagerVfs::Lazy(id)
+      },
+      DeclFs::Mod(children) => EagerVfs::Eager(
+        children.into_iter().map(|(k, v)| (k.marker(), v.to_api_rec(vfses))).collect(),
+      ),
+    }
+  }
+}

orchid-extension/src/lexer.rs

@@ -0,0 +1,34 @@
+use std::ops::RangeInclusive;
+
+use orchid_api::error::ProjResult;
+use orchid_api::tree::TokenTree;
+
+pub trait Lexer: Send + Sync + Sized + Default + 'static {
+  const CHAR_FILTER: &'static [RangeInclusive<char>];
+  fn lex<'a>(
+    tail: &'a str,
+    recur: impl FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+  ) -> Option<ProjResult<(&'a str, TokenTree)>>;
+}
+
+pub trait DynLexer: Send + Sync + 'static {
+  fn char_filter(&self) -> &'static [RangeInclusive<char>];
+  fn lex<'a>(
+    &self,
+    tail: &'a str,
+    recur: &mut dyn FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+  ) -> Option<ProjResult<(&'a str, TokenTree)>>;
+}
+
+impl<T: Lexer> DynLexer for T {
+  fn char_filter(&self) -> &'static [RangeInclusive<char>] { T::CHAR_FILTER }
+  fn lex<'a>(
+    &self,
+    tail: &'a str,
+    recur: &mut dyn FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+  ) -> Option<ProjResult<(&'a str, TokenTree)>> {
+    T::lex(tail, recur)
+  }
+}
+
+pub type LexerObj = &'static dyn DynLexer;

orchid-extension/src/lib.rs

@@ -1,5 +1,9 @@
+pub mod atom;
 pub mod entrypoint;
-pub mod data;
+pub mod expr;
+pub mod fs;
+pub mod lexer;
 pub mod msg;
-pub mod system_ctor;
+pub mod other_system;
 pub mod system;
+pub mod system_ctor;

orchid-extension/src/other_system.rs

@@ -0,0 +1,37 @@
+use std::marker::PhantomData;
+
+use orchid_api::atom::Atom;
+use orchid_api::expr::ExprTicket;
+use orchid_api::proto::ExtMsgSet;
+use orchid_api::system::SysId;
+use orchid_base::reqnot::ReqNot;
+
+use crate::atom::{decode_atom, AtomCard, ForeignAtom};
+use crate::system::SystemCard;
+
+pub struct SystemHandle<C: SystemCard> {
+  pub(crate) _card: PhantomData<C>,
+  pub(crate) id: SysId,
+  pub(crate) reqnot: ReqNot<ExtMsgSet>,
+}
+impl<T: SystemCard> SystemHandle<T> {
+  pub(crate) fn new(id: SysId, reqnot: ReqNot<ExtMsgSet>) -> Self {
+    Self { _card: PhantomData, id, reqnot }
+  }
+  pub fn id(&self) -> SysId { self.id }
+  pub fn wrap_atom<A: AtomCard<Owner = T>>(
+    &self,
+    api: Atom,
+    ticket: ExprTicket,
+  ) -> Result<ForeignAtom<A>, Atom> {
+    if api.owner == self.id {
+      if let Some(value) = decode_atom::<A>(&T::default(), &api) {
+        return Ok(ForeignAtom { ticket, sys: self.clone(), value, api });
+      }
+    }
+    Err(api)
+  }
+}
+impl<T: SystemCard> Clone for SystemHandle<T> {
+  fn clone(&self) -> Self { Self { reqnot: self.reqnot.clone(), _card: PhantomData, id: self.id } }
+}

orchid-extension/src/system.rs

@@ -1,6 +1,53 @@
-use orchid_api::expr::Expr;
+use std::any::Any;
+use std::io::{Read, Write};
 
-pub trait System: Send {
+use orchid_api::expr::ExprTicket;
-  fn consts(&self) -> Expr;
+use orchid_api::tree::TreeModule;
+use typeid::ConstTypeId;
+
+use crate::atom::AtomInfo;
+use crate::expr::GenClause;
+use crate::fs::DeclFs;
+use crate::lexer::LexerObj;
+
+/// System as consumed by foreign code
+pub trait SystemCard: Default + Send + Sync + 'static {
+  const NAME: &'static str;
+  const ATOM_DEFS: &'static [Option<AtomInfo>];
 }
 
+pub trait DynSystemCard: Send + Sync + 'static {
+  fn name(&self) -> &'static str;
+  fn atoms(&self) -> &'static [Option<AtomInfo>];
+  fn atom_info_for(&self, tid: ConstTypeId) -> Option<(usize, &AtomInfo)> {
+    (self.atoms().iter().enumerate())
+      .filter_map(|(i, o)| o.as_ref().map(|a| (i, a)))
+      .find(|ent| ent.1.tid == tid)
+  }
+}
+
+impl<T: SystemCard> DynSystemCard for T {
+  fn name(&self) -> &'static str { Self::NAME }
+  fn atoms(&self) -> &'static [Option<AtomInfo>] { Self::ATOM_DEFS }
+}
+
+/// System as defined by author
+pub trait System: Send + Sync + SystemCard {
+  fn env() -> TreeModule;
+  fn source() -> DeclFs;
+  const LEXERS: &'static [LexerObj];
+}
+
+pub trait DynSystem: Send + Sync + 'static {
+  fn env(&self) -> TreeModule;
+  fn source(&self) -> DeclFs;
+  fn lexers(&self) -> &'static [LexerObj];
+  fn card(&self) -> &dyn DynSystemCard;
+}
+
+impl<T: System> DynSystem for T {
+  fn env(&self) -> TreeModule { <Self as System>::env() }
+  fn source(&self) -> DeclFs { <Self as System>::source() }
+  fn lexers(&self) -> &'static [LexerObj] { Self::LEXERS }
+  fn card(&self) -> &dyn DynSystemCard { self }
+}

orchid-extension/src/system_ctor.rs

@@ -1,4 +1,3 @@
-use std::any::TypeId;
 use std::hash::{Hash as _, Hasher as _};
 
 use itertools::Itertools as _;
@@ -6,9 +5,10 @@ use orchid_api::proto::ExtMsgSet;
 use orchid_api::system::{NewSystem, SysId, SystemDecl};
 use orchid_base::reqnot::ReqNot;
 use ordered_float::NotNan;
+use typeid::ConstTypeId;
 
-use crate::data::{SystemDepCard, SystemHandle};
+use crate::other_system::SystemHandle;
-use crate::system::System;
+use crate::system::{DynSystem, System, SystemCard};
 
 pub struct SystemParams<Ctor: SystemCtor + ?Sized> {
   pub deps: <Ctor::Deps as DepSet>::Sat,
@@ -22,7 +22,7 @@ pub trait DepSet {
   fn create(take: &mut impl FnMut() -> SysId, reqnot: ReqNot<ExtMsgSet>) -> Self::Sat;
 }
 
-impl<T: SystemDepCard> DepSet for T {
+impl<T: SystemCard> DepSet for T {
   type Sat = SystemHandle<Self>;
   fn report(names: &mut impl FnMut(&'static str)) { names(T::NAME) }
   fn create(take: &mut impl FnMut() -> SysId, reqnot: ReqNot<ExtMsgSet>) -> Self::Sat {
@@ -32,15 +32,16 @@ impl<T: SystemDepCard> DepSet for T {
 
 pub trait SystemCtor: Send + 'static {
   type Deps: DepSet;
+  type Instance: System;
   const NAME: &'static str;
   const VERSION: f64;
   #[allow(clippy::new_ret_no_self)]
-  fn new(params: SystemParams<Self>) -> Box<dyn System>;
+  fn new(params: SystemParams<Self>) -> Self::Instance;
 }
 
 pub trait DynSystemCtor: Send + 'static {
   fn decl(&self) -> SystemDecl;
-  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn System>;
+  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn DynSystem>;
 }
 
 impl<T: SystemCtor> DynSystemCtor for T {
@@ -52,19 +53,19 @@ impl<T: SystemCtor> DynSystemCtor for T {
     T::Deps::report(&mut |n| depends.push(n.to_string()));
     // generate definitely unique id
     let mut ahash = ahash::AHasher::default();
-    TypeId::of::<T>().hash(&mut ahash);
+    ConstTypeId::of::<T>().hash(&mut ahash);
     let id = (ahash.finish().to_be_bytes().into_iter().tuples())
       .map(|(l, b)| u16::from_be_bytes([l, b]))
       .fold(0, |a, b| a ^ b);
     SystemDecl { name: T::NAME.to_string(), depends, id, priority }
   }
-  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn System> {
+  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn DynSystem> {
     let mut ids = new.depends.iter().copied();
-    T::new(SystemParams {
+    Box::new(T::new(SystemParams {
       deps: T::Deps::create(&mut || ids.next().unwrap(), reqnot.clone()),
       id: new.id,
       reqnot,
-    })
+    }))
   }
 }
 

orchid-extension/src/trait_obj_coder.rs

@@ -0,0 +1,44 @@
+pub struct TraitObject<T: ?Sized + Coding + 'static>(Box<T>, Arc<dyn Fn(&mut dyn Read) -> Self>);
+impl<T: ?Sized + Coding + 'static> TraitObject<T> {
+  fn inner_type_id(&self) -> ConstTypeId { self.0.as_ref().type_id() }
+  fn get_decoder(&self) -> Arc<dyn Fn(&mut dyn Read) -> Self> { self.1.clone() }
+}
+pub trait AsTraitObject<T: ?Sized + Coding + 'static>: 'static {
+  fn trait_box(self) -> Box<T>;
+  fn into_trait_object(self) -> TraitObject<T>
+  where Self: Sized + Coding {
+    let decoder = Self::get_decoder(Self::into_trait_object);
+    TraitObject(self.trait_box(), Arc::new(decoder))
+  }
+}
+
+pub struct TraitObjectCoder<T: ?Sized + Coding + 'static> {
+  entries: HashMap<u64, Box<dyn Fn(&mut dyn Read) -> TraitObject<T>>>,
+}
+impl<T: ?Sized + Coding + 'static> TraitObjectCoder<T> {
+  pub fn add_type<U: AsTraitObject<T> + Coding>(&mut self, tid_hash: u64) {
+    self.entries.entry(tid_hash).or_insert_with(|| Box::new(|b| U::decode(b).into_trait_object()));
+  }
+  pub fn add_obj(&mut self, tid_hash: u64, obj: &TraitObject<T>) {
+    self.entries.entry(tid_hash).or_insert_with(|| {
+      let decoder = obj.get_decoder();
+      Box::new(move |b| decoder(b))
+    });
+  }
+  pub fn encode<U: AsTraitObject<T> + Coding>(&mut self, data: U, out: &mut impl Write) {
+    let tid = hash_tid(ConstTypeId::of::<U>());
+    tid.encode(out);
+    self.add_type::<U>(tid);
+    data.encode(out);
+  }
+  pub fn encode_obj(&mut self, data: &TraitObject<T>, out: &mut impl Write) {
+    let tid = hash_tid(data.inner_type_id());
+    self.add_obj(tid, data);
+    tid.encode(out);
+    data.0.as_ref().encode(out);
+  }
+  pub fn decode(&mut self, src: &mut impl Read) -> TraitObject<T> {
+    let tid = u64::decode(src);
+    (self.entries.get(&tid).expect("Received object of unknown ConstTypeId"))(src)
+  }
+}

orchid-host/src/child.rs

@@ -1,6 +1,5 @@
-use std::io;
 use std::sync::Mutex;
-use std::{mem, process};
+use std::{io, mem, process};
 
 use orchid_base::msg::{recv_msg, send_msg};
 

orchid-host/src/extension.rs

@@ -10,12 +10,14 @@ use lazy_static::lazy_static;
 use orchid_api::atom::{Atom, AtomDrop, AtomSame, CallRef, FinalCall, Fwd, Fwded};
 use orchid_api::expr::{Acquire, Expr, ExprNotif, ExprTicket, Release, Relocate};
 use orchid_api::intern::IntReq;
+use orchid_api::parser::CharFilter;
 use orchid_api::proto::{
   ExtHostNotif, ExtHostReq, ExtensionHeader, HostExtNotif, HostHeader, HostMsgSet,
 };
 use orchid_api::system::{NewSystem, SysDeclId, SysId, SystemDecl, SystemDrop};
 use orchid_api::tree::{GetConstTree, TreeModule};
 use orchid_api_traits::{Decode, Encode};
+use orchid_base::char_filter::char_filter_match;
 use orchid_base::clone;
 use orchid_base::intern::{deintern, intern};
 use orchid_base::reqnot::{ReqNot, Requester as _};
@@ -166,11 +168,12 @@ impl SystemCtor {
     let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
     static NEXT_ID: AtomicU16 = AtomicU16::new(0);
     let id = NEXT_ID.fetch_add(1, Ordering::Relaxed);
-    let () = ext.reqnot.request(NewSystem { depends, id, system: self.decl.id });
+    let sys_inst = ext.reqnot.request(NewSystem { depends, id, system: self.decl.id });
     let data = System(Arc::new(SystemInstData {
       decl_id: self.decl.id,
       ext: Extension(ext),
       exprs: RwLock::default(),
+      lex_filter: sys_inst.lex_filter,
       id,
     }));
     inst_g.insert(id, data.clone());
@@ -187,6 +190,7 @@ pub struct SystemInstData {
   exprs: RwLock<HashMap<ExprTicket, (AtomicU32, RtExpr)>>,
   ext: Extension,
   decl_id: SysDeclId,
+  lex_filter: CharFilter,
   id: u16,
 }
 impl Drop for SystemInstData {
@@ -211,8 +215,9 @@ impl System {
       .or_insert((AtomicU32::new(1), get_expr()));
     ticket
   }
-
   pub fn const_tree(&self) -> TreeModule { self.0.ext.0.reqnot.request(GetConstTree(self.0.id)) }
+  pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
+  pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
 }
 impl fmt::Debug for System {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {