Finally figured out how I want atoms to work

orchid-extension/src/atom.rs

@@ -0,0 +1,174 @@
+use std::any::{type_name, Any};
+use std::io::{Read, Write};
+use std::num::NonZeroU64;
+use std::ops::Deref;
+use std::sync::Arc;
+use std::{fmt, mem};
+
+use derive_destructure::destructure;
+use orchid_api::atom::{Atom, Fwd};
+use orchid_api::expr::{ExprTicket, Release};
+use orchid_api::system::SysId;
+use orchid_api_traits::{Coding, Decode, Encode, Request};
+use orchid_base::id_store::{IdRecord, IdStore};
+use orchid_base::reqnot::Requester as _;
+use typeid::ConstTypeId;
+
+use crate::expr::GenClause;
+use crate::other_system::SystemHandle;
+use crate::system::{DynSystemCard, SystemCard};
+
+pub trait AtomCard: 'static + Sized {
+  type Owner: SystemCard;
+  type Data: Clone + Coding + Sized;
+  type Req: Coding;
+}
+
+pub fn get_info<A: AtomCard>(sys: &(impl DynSystemCard + ?Sized)) -> (usize, &AtomInfo) {
+  sys.atom_info_for(ConstTypeId::of::<A>()).unwrap_or_else(|| {
+    panic!("Atom {} not associated with system {}", type_name::<A>(), sys.name())
+  })
+}
+
+pub fn encode_atom_nodrop<A: AtomCard>(
+  sys_id: SysId,
+  sys: &(impl DynSystemCard + ?Sized),
+  data: &A::Data,
+) -> Atom {
+  let (info_pos, _) = get_info::<A>(sys);
+  let mut buf = (info_pos as u64).enc_vec();
+  data.encode(&mut buf);
+  Atom { owner: sys_id, drop: false, data: buf }
+}
+
+pub fn encode_atom_drop<A: AtomCard>(
+  sys_id: SysId,
+  sys: &(impl DynSystemCard + ?Sized),
+  atom_id: u64,
+  data: &A::Data,
+) -> Atom {
+  let (info_pos, _) = get_info::<A>(sys);
+  let mut buf = (info_pos as u64).enc_vec();
+  atom_id.encode(&mut buf);
+  data.encode(&mut buf);
+  Atom { owner: sys_id, drop: true, data: buf }
+}
+
+pub fn decode_atom<A: AtomCard>(
+  sys: &(impl DynSystemCard + ?Sized),
+  Atom { data, drop, owner: _ }: &Atom,
+) -> Option<A::Data> {
+  let (info_pos, info) = get_info::<A>(sys);
+  let mut data = &data[..];
+  if u64::decode(&mut data) != info_pos as u64 {
+    return None;
+  }
+  let val = (info.decode)(data);
+  Some(*val.downcast().expect("The type-id checked out, the decode should've worked"))
+}
+
+#[derive(destructure)]
+pub struct ForeignAtom<A: AtomCard> {
+  pub(crate) sys: SystemHandle<A::Owner>,
+  pub(crate) ticket: ExprTicket,
+  pub(crate) api: Atom,
+  pub(crate) value: A::Data,
+}
+impl<A: AtomCard> ForeignAtom<A> {
+  /// Unpack the object, returning the held atom and expr ticket. This is in
+  /// contrast to dropping the atom which releases the expr ticket.
+  pub fn unpack(self) -> (A::Data, ExprTicket, Atom) {
+    let (_, ticket, api, value) = self.destructure();
+    (value, ticket, api)
+  }
+  pub fn ticket(&self) -> ExprTicket { self.ticket }
+  pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
+    R::Response::decode(&mut &self.sys.reqnot.request(Fwd(self.api.clone(), req.enc_vec()))[..])
+  }
+}
+impl<A: AtomCard> Deref for ForeignAtom<A> {
+  type Target = A::Data;
+  fn deref(&self) -> &Self::Target { &self.value }
+}
+impl<A: AtomCard> Drop for ForeignAtom<A> {
+  fn drop(&mut self) { self.sys.reqnot.notify(Release(self.sys.id(), self.ticket)) }
+}
+
+pub struct AtomInfo {
+  pub tid: ConstTypeId,
+  pub decode: fn(&[u8]) -> Box<dyn Any>,
+  pub call: fn(&[u8], ExprTicket) -> GenClause,
+  pub call_ref: fn(&[u8], ExprTicket) -> GenClause,
+  pub same: fn(&[u8], &[u8]) -> bool,
+  pub handle_req: fn(&[u8], &mut dyn Read, &mut dyn Write),
+  pub drop: fn(&[u8]),
+}
+
+pub trait ThinAtom: AtomCard<Data = Self> + Coding + fmt::Debug {
+  fn call(&self, arg: ExprTicket) -> GenClause;
+  fn same(&self, other: &Self) -> bool;
+  fn handle_req(&self, req: Self::Req, rep: &mut (impl Write + ?Sized));
+}
+
+pub const fn thin_atom_info<T: ThinAtom>() -> AtomInfo {
+  AtomInfo {
+    tid: ConstTypeId::of::<T>(),
+    decode: |mut b| Box::new(T::decode(&mut b)),
+    call: |mut b, extk| T::decode(&mut b).call(extk),
+    call_ref: |mut b, extk| T::decode(&mut b).call(extk),
+    handle_req: |mut b, req, rep| T::decode(&mut b).handle_req(Decode::decode(req), rep),
+    same: |mut b1, mut b2| T::decode(&mut b1).same(&T::decode(&mut b2)),
+    drop: |mut b1| eprintln!("Received drop signal for non-drop atom {:?}", T::decode(&mut b1)),
+  }
+}
+
+/// Atoms that have a [Drop]
+pub trait OwnedAtom: AtomCard + Deref<Target = Self::Data> + Send + Sync + Any + 'static {
+  fn call_ref(&self, arg: ExprTicket) -> GenClause;
+  fn call(self, arg: ExprTicket) -> GenClause;
+  fn same(&self, other: &Self) -> bool;
+  fn handle_req(&self, req: Self::Req, rep: &mut (impl Write + ?Sized));
+}
+
+pub trait DynOwnedAtom: Send + Sync + 'static {
+  fn atom_tid(&self) -> ConstTypeId;
+  fn as_any_ref(&self) -> &dyn Any;
+  fn dyn_call_ref(&self, arg: ExprTicket) -> GenClause;
+  fn dyn_call(self: Box<Self>, arg: ExprTicket) -> GenClause;
+  fn dyn_same(&self, other: &dyn DynOwnedAtom) -> bool;
+  fn dyn_handle_req(&self, req: &mut dyn Read, rep: &mut dyn Write);
+}
+
+impl<T: OwnedAtom> DynOwnedAtom for T {
+  fn atom_tid(&self) -> ConstTypeId { ConstTypeId::of::<T>() }
+  fn as_any_ref(&self) -> &dyn Any { self }
+  fn dyn_call_ref(&self, arg: ExprTicket) -> GenClause { self.call_ref(arg) }
+  fn dyn_call(self: Box<Self>, arg: ExprTicket) -> GenClause { self.call(arg) }
+  fn dyn_same(&self, other: &dyn DynOwnedAtom) -> bool {
+    if ConstTypeId::of::<Self>() != other.as_any_ref().type_id() {
+      return false;
+    }
+    let other_self = other.as_any_ref().downcast_ref().expect("The type_ids are the same");
+    self.same(other_self)
+  }
+  fn dyn_handle_req(&self, req: &mut dyn Read, rep: &mut dyn Write) {
+    self.handle_req(<Self as AtomCard>::Req::decode(req), rep)
+  }
+}
+
+pub(crate) static OBJ_STORE: IdStore<Box<dyn DynOwnedAtom>> = IdStore::new();
+
+const fn owned_atom_info<T: OwnedAtom>() -> AtomInfo {
+  fn with_atom<U>(mut b: &[u8], f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>) -> U) -> U {
+    f(OBJ_STORE.get(NonZeroU64::decode(&mut b)).expect("Received invalid atom ID"))
+  }
+  AtomInfo {
+    tid: ConstTypeId::of::<T>(),
+    decode: |mut b| Box::new(T::Data::decode(&mut b)),
+    call: |b, arg| with_atom(b, |a| a.remove().dyn_call(arg)),
+    call_ref: |b, arg| with_atom(b, |a| a.dyn_call_ref(arg)),
+    same: |b1, b2| with_atom(b1, |a1| with_atom(b2, |a2| a1.dyn_same(&**a2))),
+    handle_req: |b, req, rep| with_atom(b, |a| a.dyn_handle_req(req, rep)),
+    drop: |b| mem::drop(with_atom(b, |a| a.remove())),
+  }
+}

orchid-extension/src/data.rs

@@ -1,83 +0,0 @@
-use std::marker::PhantomData;
-use std::ops::Deref;
-
-use derive_destructure::destructure;
-use orchid_api::atom::{Atom, Fwd};
-use orchid_api::expr::{ExprTicket, Release};
-use orchid_api::proto::ExtMsgSet;
-use orchid_api::system::SysId;
-use orchid_api_traits::{Coding, Decode, Request};
-use orchid_base::reqnot::{ReqNot, Requester};
-
-pub struct SystemHandle<T: SystemDepCard> {
-  _t: PhantomData<T>,
-  id: SysId,
-  reqnot: ReqNot<ExtMsgSet>,
-}
-impl<T: SystemDepCard> SystemHandle<T> {
-  pub(crate) fn new(id: SysId, reqnot: ReqNot<ExtMsgSet>) -> Self {
-    Self { _t: PhantomData, id, reqnot }
-  }
-  pub fn id(&self) -> SysId { self.id }
-  pub fn wrap_atom<A: Atomic<Owner = T>>(
-    &self,
-    api: Atom,
-    ticket: ExprTicket,
-  ) -> Result<OwnedAtom<A>, Atom> {
-    if api.owner == self.id {
-      Ok(OwnedAtom { ticket, sys: self.clone(), value: T::decode_atom(&api), api })
-    } else {
-      Err(api)
-    }
-  }
-}
-impl<T: SystemDepCard> Clone for SystemHandle<T> {
-  fn clone(&self) -> Self { Self { reqnot: self.reqnot.clone(), _t: PhantomData, id: self.id } }
-}
-
-pub trait Atomic: 'static {
-  type Owner: SystemDepCard;
-  type Req: Coding;
-  const HAS_DROP: bool;
-}
-
-pub trait SystemDepCard: 'static {
-  const NAME: &'static str;
-  /// Decode an atom from binary representation.
-  ///
-  /// This is held in the dep card because there is no global type tag on the
-  /// atom, so by the logic of the binary coding algorithm the value has to be a
-  /// concrete type, probably an enum of the viable types.
-  fn decode_atom<A: Atomic<Owner = Self>>(api: &Atom) -> A;
-}
-
-#[derive(destructure)]
-pub struct OwnedAtom<A: Atomic> {
-  sys: SystemHandle<A::Owner>,
-  ticket: ExprTicket,
-  api: Atom,
-  value: A,
-}
-impl<A: Atomic> OwnedAtom<A> {
-  /// Unpack the object, returning the held atom and expr ticket. This is in
-  /// contrast to dropping the atom which releases the expr ticket.
-  pub fn unpack(self) -> (A, ExprTicket, Atom) {
-    let (_, ticket, api, value) = self.destructure();
-    (value, ticket, api)
-  }
-  pub fn ticket(&self) -> ExprTicket { self.ticket }
-  pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
-    R::Response::decode(&mut &self.sys.reqnot.request(Fwd(self.api.clone(), req.enc_vec()))[..])
-  }
-}
-impl<A: Atomic> Deref for OwnedAtom<A> {
-  type Target = A;
-  fn deref(&self) -> &Self::Target { &self.value }
-}
-impl<A: Atomic> Drop for OwnedAtom<A> {
-  fn drop(&mut self) {
-    if A::HAS_DROP {
-      self.sys.reqnot.notify(Release(self.sys.id(), self.ticket))
-    }
-  }
-}

orchid-extension/src/entrypoint.rs

@@ -1,34 +1,65 @@
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
+use std::{mem, thread};
 
 use hashbrown::HashMap;
 use itertools::Itertools;
+use orchid_api::atom::{Atom, AtomReq, AtomSame, CallRef, FinalCall, Fwded};
+use orchid_api::parser::{CharFilter, Lex, Lexed, ParserReq, SubLex};
 use orchid_api::proto::{ExtMsgSet, ExtensionHeader, HostExtNotif, HostExtReq, HostHeader};
+use orchid_api::system::{SysId, SystemInst};
+use orchid_api::vfs::{EagerVfs, VfsId, VfsRead, VfsReq};
 use orchid_api_traits::{Decode, Encode};
+use orchid_base::char_filter::{char_filter_union, mk_char_filter};
 use orchid_base::clone;
-use orchid_base::intern::{init_replica, sweep_replica};
+use orchid_base::intern::{deintern, init_replica, sweep_replica};
+use orchid_base::name::PathSlice;
 use orchid_base::reqnot::{ReqNot, Requester};
 
+use crate::atom::AtomInfo;
+use crate::fs::VirtFS;
 use crate::msg::{recv_parent_msg, send_parent_msg};
+use crate::system::DynSystem;
 use crate::system_ctor::DynSystemCtor;
 
 pub struct ExtensionData {
   pub systems: Vec<Box<dyn DynSystemCtor>>,
 }
 
+pub struct SystemRecord {
+  instance: Box<dyn DynSystem>,
+  vfses: HashMap<VfsId, Arc<dyn VirtFS>>,
+  declfs: EagerVfs,
+}
+
+pub fn with_atom_record<T>(
+  systems: &Mutex<HashMap<SysId, SystemRecord>>,
+  atom: &Atom,
+  cb: impl FnOnce(&AtomInfo, &[u8]) -> T,
+) -> T {
+  let mut data = &atom.data[..];
+  let systems_g = systems.lock().unwrap();
+  let sys = &systems_g[&atom.owner].instance;
+  let atom_record =
+    (sys.card().atoms()[u64::decode(&mut data) as usize].as_ref()).expect("Atom ID reserved");
+  cb(atom_record, data)
+}
+
 pub fn main(data: ExtensionData) {
   HostHeader::decode(&mut &recv_parent_msg().unwrap()[..]);
   let mut buf = Vec::new();
   let decls = data.systems.iter().map(|sys| sys.decl()).collect_vec();
-  let systems = Arc::new(Mutex::new(HashMap::new()));
+  let systems = Arc::new(Mutex::new(HashMap::<SysId, SystemRecord>::new()));
   ExtensionHeader { systems: decls.clone() }.encode(&mut buf);
   send_parent_msg(&buf).unwrap();
   let exiting = Arc::new(AtomicBool::new(false));
   let rn = ReqNot::<ExtMsgSet>::new(
     |a, _| send_parent_msg(a).unwrap(),
-    clone!(exiting; move |n, _| match n {
+    clone!(systems, exiting; move |n, _| match n {
       HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
-      _ => todo!(),
+      HostExtNotif::SystemDrop(sys) => mem::drop(systems.lock().unwrap().remove(&sys.0)),
+      HostExtNotif::AtomDrop(atom) =>
+        with_atom_record(&systems, &atom.0, |rec, data| (rec.drop)(data)),
     }),
     clone!(systems; move |req| match req.req() {
       HostExtReq::Ping(ping) => req.handle(ping, &()),
@@ -36,10 +67,57 @@ pub fn main(data: ExtensionData) {
       HostExtReq::NewSystem(new_sys) => {
         let i = decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system).unwrap().0;
         let system = data.systems[i].new_system(new_sys, req.reqnot());
-        systems.lock().unwrap().insert(new_sys.id, system);
-        req.handle(new_sys, &())
+        let mut vfses = HashMap::new();
+        let lex_filter = system.lexers().iter().fold(CharFilter(vec![]), |cf, lx| {
+          char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
+        });
+        systems.lock().unwrap().insert(new_sys.id, SystemRecord {
+          declfs: system.source().to_api_rec(&mut vfses),
+          vfses,
+          instance: system,
+        });
+        req.handle(new_sys, &SystemInst {
+          lex_filter
+        })
+      }
+      HostExtReq::GetConstTree(get_tree) => {
+        let systems_g = systems.lock().unwrap();
+        req.handle(get_tree, &systems_g[&get_tree.0].instance.env())
+      }
+      HostExtReq::VfsReq(VfsReq::GetVfs(get_vfs)) => {
+        let systems_g = systems.lock().unwrap();
+        req.handle(get_vfs, &systems_g[&get_vfs.0].declfs)
+      }
+      HostExtReq::VfsReq(VfsReq::VfsRead(vfs_read@VfsRead(sys_id, vfs_id, path))) => {
+        let systems_g = systems.lock().unwrap();
+        let path = path.iter().map(|t| deintern(*t)).collect_vec();
+        req.handle(vfs_read, &systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path)))
+      }
+      HostExtReq::ParserReq(ParserReq::Lex(lex)) => {
+        let systems_g = systems.lock().unwrap();
+        let Lex{ sys, text, pos } = *lex;
+        let lexers = systems_g[&sys].instance.lexers();
+        mem::drop(systems_g);
+        let source = deintern(text);
+        let tk = req.will_handle_as(lex);
+        thread::spawn(move || {
+          let reqnot = req.reqnot();
+          let mut recurse = |tail: &str| {
+            let pos = (source.len() - tail.len()) as u32;
+            let lexed = reqnot.request(SubLex{ pos, text })?;
+            Ok((&source[lexed.pos as usize..], lexed.data))
+          };
+          let lex_res = lexers.iter().find_map(|lx| lx.lex(&source[pos as usize..], &mut recurse));
+          req.handle_as(tk, &lex_res.map(|r| r.map(|(s, data)| {
+            let pos = (source.len() - s.len()) as u32;
+            Lexed { data, pos }
+          })))
+        });
       },
-      _ => todo!(),
+      HostExtReq::AtomReq(AtomReq::AtomSame(same@AtomSame(l, r))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(AtomReq::Fwded(call@Fwded(atom, req))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(AtomReq::CallRef(call@CallRef(atom, arg))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(AtomReq::FinalCall(call@FinalCall(atom, arg))) => todo!("subsys nimpl"),
     }),
   );
   init_replica(rn.clone().map());

orchid-extension/src/expr.rs

@@ -0,0 +1,53 @@
+use orchid_api::atom::Atom;
+use orchid_api::expr::ExprTicket;
+use orchid_api::system::SysId;
+use orchid_base::id_store::IdStore;
+use orchid_base::intern::Token;
+
+use crate::atom::{encode_atom_nodrop, DynOwnedAtom, OwnedAtom, ThinAtom, OBJ_STORE};
+use crate::system::DynSystem;
+
+pub enum GenClause {
+  Call(Box<GenClause>, Box<GenClause>),
+  Lambda(Token<String>, Box<GenClause>),
+  Arg(Token<String>),
+  Slot(ExprTicket),
+  Seq(Box<GenClause>, Box<GenClause>),
+  Const(Token<Vec<Token<String>>>),
+  ThinAtom(Box<dyn Fn(SysId, &dyn DynSystem) -> Atom>),
+  OwnedAtom(u64),
+}
+
+pub fn cnst(path: Token<Vec<Token<String>>>) -> GenClause { GenClause::Const(path) }
+pub fn val<A: ThinAtom>(atom: A) -> GenClause {
+  GenClause::ThinAtom(Box::new(move |id, sys| encode_atom_nodrop::<A>(id, sys.card(), &atom)))
+}
+
+pub fn obj<A: OwnedAtom>(atom: A) -> GenClause {
+  GenClause::OwnedAtom(OBJ_STORE.add(Box::new(atom)))
+}
+
+pub fn seq(ops: impl IntoIterator<Item = GenClause>) -> GenClause {
+  fn recur(mut ops: impl Iterator<Item = GenClause>) -> Option<GenClause> {
+    let op = ops.next()?;
+    Some(match recur(ops) {
+      None => op,
+      Some(rec) => GenClause::Seq(Box::new(op), Box::new(rec)),
+    })
+  }
+  recur(ops.into_iter()).expect("Empty list provided to seq!")
+}
+
+pub fn slot(extk: ExprTicket) -> GenClause { GenClause::Slot(extk) }
+
+pub fn arg(n: Token<String>) -> GenClause { GenClause::Arg(n) }
+
+pub fn lambda(n: Token<String>, b: impl IntoIterator<Item = GenClause>) -> GenClause {
+  GenClause::Lambda(n, Box::new(call(b)))
+}
+
+pub fn call(v: impl IntoIterator<Item = GenClause>) -> GenClause {
+  v.into_iter()
+    .reduce(|f, x| GenClause::Call(Box::new(f), Box::new(x)))
+    .expect("Empty call expression")
+}

orchid-extension/src/fs.rs

@@ -0,0 +1,48 @@
+use std::sync::Arc;
+
+use hashbrown::HashMap;
+use orchid_api::error::ProjResult;
+use orchid_api::vfs::{EagerVfs, Loaded, VfsId};
+use orchid_base::intern::{intern, Token};
+use orchid_base::name::PathSlice;
+use substack::Substack;
+use trait_set::trait_set;
+
+pub trait VirtFS: Send + Sync + 'static {
+  fn load(&self, path: &PathSlice) -> ProjResult<Loaded>;
+}
+
+trait_set! {
+  pub trait RecFsHandler<E> = FnMut(Substack<Token<String>>, &Arc<dyn VirtFS>) -> Result<(), E>;
+}
+
+pub enum DeclFs {
+  Lazy(Arc<dyn VirtFS>),
+  Mod(HashMap<Token<String>, DeclFs>),
+}
+impl DeclFs {
+  pub fn module(entries: impl IntoIterator<Item = (&'static str, Self)>) -> Self {
+    Self::Mod(entries.into_iter().map(|(k, v)| (intern(k), v)).collect())
+  }
+  fn rec<E>(&self, path: Substack<Token<String>>, f: &mut impl RecFsHandler<E>) -> Result<(), E> {
+    match self {
+      DeclFs::Lazy(fs) => f(path, fs),
+      DeclFs::Mod(entries) => entries.iter().try_for_each(|(k, v)| v.rec(path.push(k.clone()), f)),
+    }
+  }
+  pub fn recurse<E>(&self, f: &mut impl RecFsHandler<E>) -> Result<(), E> {
+    self.rec(Substack::Bottom, f)
+  }
+  pub fn to_api_rec(&self, vfses: &mut HashMap<VfsId, Arc<dyn VirtFS>>) -> EagerVfs {
+    match self {
+      DeclFs::Lazy(fs) => {
+        let id = vfses.len() as VfsId;
+        vfses.insert(id, fs.clone());
+        EagerVfs::Lazy(id)
+      },
+      DeclFs::Mod(children) => EagerVfs::Eager(
+        children.into_iter().map(|(k, v)| (k.marker(), v.to_api_rec(vfses))).collect(),
+      ),
+    }
+  }
+}

orchid-extension/src/lexer.rs

@@ -0,0 +1,34 @@
+use std::ops::RangeInclusive;
+
+use orchid_api::error::ProjResult;
+use orchid_api::tree::TokenTree;
+
+pub trait Lexer: Send + Sync + Sized + Default + 'static {
+  const CHAR_FILTER: &'static [RangeInclusive<char>];
+  fn lex<'a>(
+    tail: &'a str,
+    recur: impl FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+  ) -> Option<ProjResult<(&'a str, TokenTree)>>;
+}
+
+pub trait DynLexer: Send + Sync + 'static {
+  fn char_filter(&self) -> &'static [RangeInclusive<char>];
+  fn lex<'a>(
+    &self,
+    tail: &'a str,
+    recur: &mut dyn FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+  ) -> Option<ProjResult<(&'a str, TokenTree)>>;
+}
+
+impl<T: Lexer> DynLexer for T {
+  fn char_filter(&self) -> &'static [RangeInclusive<char>] { T::CHAR_FILTER }
+  fn lex<'a>(
+    &self,
+    tail: &'a str,
+    recur: &mut dyn FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+  ) -> Option<ProjResult<(&'a str, TokenTree)>> {
+    T::lex(tail, recur)
+  }
+}
+
+pub type LexerObj = &'static dyn DynLexer;

orchid-extension/src/lib.rs

@@ -1,5 +1,9 @@
+pub mod atom;
 pub mod entrypoint;
-pub mod data;
+pub mod expr;
+pub mod fs;
+pub mod lexer;
 pub mod msg;
-pub mod system_ctor;
+pub mod other_system;
 pub mod system;
+pub mod system_ctor;

orchid-extension/src/other_system.rs

@@ -0,0 +1,37 @@
+use std::marker::PhantomData;
+
+use orchid_api::atom::Atom;
+use orchid_api::expr::ExprTicket;
+use orchid_api::proto::ExtMsgSet;
+use orchid_api::system::SysId;
+use orchid_base::reqnot::ReqNot;
+
+use crate::atom::{decode_atom, AtomCard, ForeignAtom};
+use crate::system::SystemCard;
+
+pub struct SystemHandle<C: SystemCard> {
+  pub(crate) _card: PhantomData<C>,
+  pub(crate) id: SysId,
+  pub(crate) reqnot: ReqNot<ExtMsgSet>,
+}
+impl<T: SystemCard> SystemHandle<T> {
+  pub(crate) fn new(id: SysId, reqnot: ReqNot<ExtMsgSet>) -> Self {
+    Self { _card: PhantomData, id, reqnot }
+  }
+  pub fn id(&self) -> SysId { self.id }
+  pub fn wrap_atom<A: AtomCard<Owner = T>>(
+    &self,
+    api: Atom,
+    ticket: ExprTicket,
+  ) -> Result<ForeignAtom<A>, Atom> {
+    if api.owner == self.id {
+      if let Some(value) = decode_atom::<A>(&T::default(), &api) {
+        return Ok(ForeignAtom { ticket, sys: self.clone(), value, api });
+      }
+    }
+    Err(api)
+  }
+}
+impl<T: SystemCard> Clone for SystemHandle<T> {
+  fn clone(&self) -> Self { Self { reqnot: self.reqnot.clone(), _card: PhantomData, id: self.id } }
+}

orchid-extension/src/system.rs

@@ -1,6 +1,53 @@
-use orchid_api::expr::Expr;
+use std::any::Any;
+use std::io::{Read, Write};
 
-pub trait System: Send {
-  fn consts(&self) -> Expr;
+use orchid_api::expr::ExprTicket;
+use orchid_api::tree::TreeModule;
+use typeid::ConstTypeId;
+
+use crate::atom::AtomInfo;
+use crate::expr::GenClause;
+use crate::fs::DeclFs;
+use crate::lexer::LexerObj;
+
+/// System as consumed by foreign code
+pub trait SystemCard: Default + Send + Sync + 'static {
+  const NAME: &'static str;
+  const ATOM_DEFS: &'static [Option<AtomInfo>];
 }
 
+pub trait DynSystemCard: Send + Sync + 'static {
+  fn name(&self) -> &'static str;
+  fn atoms(&self) -> &'static [Option<AtomInfo>];
+  fn atom_info_for(&self, tid: ConstTypeId) -> Option<(usize, &AtomInfo)> {
+    (self.atoms().iter().enumerate())
+      .filter_map(|(i, o)| o.as_ref().map(|a| (i, a)))
+      .find(|ent| ent.1.tid == tid)
+  }
+}
+
+impl<T: SystemCard> DynSystemCard for T {
+  fn name(&self) -> &'static str { Self::NAME }
+  fn atoms(&self) -> &'static [Option<AtomInfo>] { Self::ATOM_DEFS }
+}
+
+/// System as defined by author
+pub trait System: Send + Sync + SystemCard {
+  fn env() -> TreeModule;
+  fn source() -> DeclFs;
+  const LEXERS: &'static [LexerObj];
+}
+
+pub trait DynSystem: Send + Sync + 'static {
+  fn env(&self) -> TreeModule;
+  fn source(&self) -> DeclFs;
+  fn lexers(&self) -> &'static [LexerObj];
+  fn card(&self) -> &dyn DynSystemCard;
+}
+
+impl<T: System> DynSystem for T {
+  fn env(&self) -> TreeModule { <Self as System>::env() }
+  fn source(&self) -> DeclFs { <Self as System>::source() }
+  fn lexers(&self) -> &'static [LexerObj] { Self::LEXERS }
+  fn card(&self) -> &dyn DynSystemCard { self }
+}

orchid-extension/src/system_ctor.rs

@@ -1,4 +1,3 @@
-use std::any::TypeId;
 use std::hash::{Hash as _, Hasher as _};
 
 use itertools::Itertools as _;
@@ -6,9 +5,10 @@ use orchid_api::proto::ExtMsgSet;
 use orchid_api::system::{NewSystem, SysId, SystemDecl};
 use orchid_base::reqnot::ReqNot;
 use ordered_float::NotNan;
+use typeid::ConstTypeId;
 
-use crate::data::{SystemDepCard, SystemHandle};
-use crate::system::System;
+use crate::other_system::SystemHandle;
+use crate::system::{DynSystem, System, SystemCard};
 
 pub struct SystemParams<Ctor: SystemCtor + ?Sized> {
   pub deps: <Ctor::Deps as DepSet>::Sat,
@@ -22,7 +22,7 @@ pub trait DepSet {
   fn create(take: &mut impl FnMut() -> SysId, reqnot: ReqNot<ExtMsgSet>) -> Self::Sat;
 }
 
-impl<T: SystemDepCard> DepSet for T {
+impl<T: SystemCard> DepSet for T {
   type Sat = SystemHandle<Self>;
   fn report(names: &mut impl FnMut(&'static str)) { names(T::NAME) }
   fn create(take: &mut impl FnMut() -> SysId, reqnot: ReqNot<ExtMsgSet>) -> Self::Sat {
@@ -32,15 +32,16 @@ impl<T: SystemDepCard> DepSet for T {
 
 pub trait SystemCtor: Send + 'static {
   type Deps: DepSet;
+  type Instance: System;
   const NAME: &'static str;
   const VERSION: f64;
   #[allow(clippy::new_ret_no_self)]
-  fn new(params: SystemParams<Self>) -> Box<dyn System>;
+  fn new(params: SystemParams<Self>) -> Self::Instance;
 }
 
 pub trait DynSystemCtor: Send + 'static {
   fn decl(&self) -> SystemDecl;
-  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn System>;
+  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn DynSystem>;
 }
 
 impl<T: SystemCtor> DynSystemCtor for T {
@@ -52,19 +53,19 @@ impl<T: SystemCtor> DynSystemCtor for T {
     T::Deps::report(&mut |n| depends.push(n.to_string()));
     // generate definitely unique id
     let mut ahash = ahash::AHasher::default();
-    TypeId::of::<T>().hash(&mut ahash);
+    ConstTypeId::of::<T>().hash(&mut ahash);
     let id = (ahash.finish().to_be_bytes().into_iter().tuples())
       .map(|(l, b)| u16::from_be_bytes([l, b]))
       .fold(0, |a, b| a ^ b);
     SystemDecl { name: T::NAME.to_string(), depends, id, priority }
   }
-  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn System> {
+  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn DynSystem> {
     let mut ids = new.depends.iter().copied();
-    T::new(SystemParams {
+    Box::new(T::new(SystemParams {
       deps: T::Deps::create(&mut || ids.next().unwrap(), reqnot.clone()),
       id: new.id,
       reqnot,
-    })
+    }))
   }
 }
 

orchid-extension/src/trait_obj_coder.rs

@@ -0,0 +1,44 @@
+pub struct TraitObject<T: ?Sized + Coding + 'static>(Box<T>, Arc<dyn Fn(&mut dyn Read) -> Self>);
+impl<T: ?Sized + Coding + 'static> TraitObject<T> {
+  fn inner_type_id(&self) -> ConstTypeId { self.0.as_ref().type_id() }
+  fn get_decoder(&self) -> Arc<dyn Fn(&mut dyn Read) -> Self> { self.1.clone() }
+}
+pub trait AsTraitObject<T: ?Sized + Coding + 'static>: 'static {
+  fn trait_box(self) -> Box<T>;
+  fn into_trait_object(self) -> TraitObject<T>
+  where Self: Sized + Coding {
+    let decoder = Self::get_decoder(Self::into_trait_object);
+    TraitObject(self.trait_box(), Arc::new(decoder))
+  }
+}
+
+pub struct TraitObjectCoder<T: ?Sized + Coding + 'static> {
+  entries: HashMap<u64, Box<dyn Fn(&mut dyn Read) -> TraitObject<T>>>,
+}
+impl<T: ?Sized + Coding + 'static> TraitObjectCoder<T> {
+  pub fn add_type<U: AsTraitObject<T> + Coding>(&mut self, tid_hash: u64) {
+    self.entries.entry(tid_hash).or_insert_with(|| Box::new(|b| U::decode(b).into_trait_object()));
+  }
+  pub fn add_obj(&mut self, tid_hash: u64, obj: &TraitObject<T>) {
+    self.entries.entry(tid_hash).or_insert_with(|| {
+      let decoder = obj.get_decoder();
+      Box::new(move |b| decoder(b))
+    });
+  }
+  pub fn encode<U: AsTraitObject<T> + Coding>(&mut self, data: U, out: &mut impl Write) {
+    let tid = hash_tid(ConstTypeId::of::<U>());
+    tid.encode(out);
+    self.add_type::<U>(tid);
+    data.encode(out);
+  }
+  pub fn encode_obj(&mut self, data: &TraitObject<T>, out: &mut impl Write) {
+    let tid = hash_tid(data.inner_type_id());
+    self.add_obj(tid, data);
+    tid.encode(out);
+    data.0.as_ref().encode(out);
+  }
+  pub fn decode(&mut self, src: &mut impl Read) -> TraitObject<T> {
+    let tid = u64::decode(src);
+    (self.entries.get(&tid).expect("Received object of unknown ConstTypeId"))(src)
+  }
+}