Lexer test mode works

Cargo.lock

@@ -32,6 +32,55 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
 
+[[package]]
+name = "anstream"
+version = "0.6.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
+dependencies = [
+ "anstyle",
+ "windows-sys",
+]
+
 [[package]]
 name = "arrayvec"
 version = "0.7.4"
@@ -117,6 +166,12 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952"
 
+[[package]]
+name = "camino"
+version = "1.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -129,6 +184,52 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
+[[package]]
+name = "clap"
+version = "4.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim 0.11.1",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64"
+dependencies = [
+ "heck",
+ "proc-macro2 1.0.78",
+ "quote 1.0.35",
+ "syn 2.0.52",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+
 [[package]]
 name = "const_panic"
 version = "0.2.8"
@@ -180,7 +281,7 @@ dependencies = [
  "ident_case",
  "proc-macro2 1.0.78",
  "quote 1.0.35",
- "strsim",
+ "strsim 0.10.0",
  "syn 2.0.52",
 ]
 
@@ -299,6 +400,12 @@ dependencies = [
  "allocator-api2",
 ]
 
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -315,6 +422,12 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -481,6 +594,7 @@ dependencies = [
  "orchid-api-traits",
  "orchid-base",
  "ordered-float",
+ "substack",
 ]
 
 [[package]]
@@ -501,6 +615,13 @@ dependencies = [
 [[package]]
 name = "orcx"
 version = "0.1.0"
+dependencies = [
+ "camino",
+ "clap",
+ "itertools",
+ "orchid-base",
+ "orchid-host",
+]
 
 [[package]]
 name = "ordered-float"
@@ -829,6 +950,12 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "substack"
 version = "1.1.0"
@@ -962,6 +1089,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "uuid"
 version = "1.10.0"
@@ -1021,6 +1154,79 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
 [[package]]
 name = "winnow"
 version = "0.5.40"

orchid-api-derive/src/decode.rs

@@ -20,7 +20,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 
 fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream {
   match fields {
-    syn::Fields::Unit => pm2::TokenStream::new(),
+    syn::Fields::Unit => quote! { },
     syn::Fields::Named(_) => {
       let names = fields.iter().map(|f| f.ident.as_ref().unwrap());
       quote! { { #( #names: orchid_api_traits::Decode::decode(read), )* } }

orchid-api-derive/src/encode.rs

@@ -37,7 +37,7 @@ fn encode_body(data: &syn::Data) -> Option<pm2::TokenStream> {
         let body = encode_items(&v.fields);
         quote! {
           Self::#ident #dest => {
-            (#i as u64).encode(write);
+            (#i as u8).encode(write);
             #body
           }
         }

orchid-api-traits/src/relations.rs

@@ -10,7 +10,7 @@ pub trait Channel: 'static {
   type Notif: Coding + Sized + Send + 'static;
 }
 
-pub trait MsgSet {
+pub trait MsgSet: Send + Sync + 'static {
   type In: Channel;
   type Out: Channel;
 }

orchid-api/src/atom.rs

@@ -104,6 +104,13 @@ impl Request for Command {
 #[extends(HostExtNotif)]
 pub struct AtomDrop(pub Atom);
 
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct AtomPrint(pub Atom);
+impl Request for AtomPrint {
+  type Response = String;
+}
+
 /// Requests that apply to an existing atom instance
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(HostExtReq)]
@@ -114,6 +121,7 @@ pub enum AtomReq {
   AtomSame(AtomSame),
   Fwded(Fwded),
   Command(Command),
+  AtomPrint(AtomPrint),
 }
 impl AtomReq {
   /// Obtain the first [Atom] argument of the request. All requests in this
@@ -124,7 +132,8 @@ impl AtomReq {
       | Self::CallRef(CallRef(a, ..))
       | Self::Command(Command(a))
       | Self::FinalCall(FinalCall(a, ..))
-      | Self::Fwded(Fwded(a, ..)) => a,
+      | Self::Fwded(Fwded(a, ..))
+      | Self::AtomPrint(AtomPrint(a)) => a,
     }
   }
 }

orchid-api/src/lib.rs

@@ -8,3 +8,4 @@ pub mod proto;
 pub mod system;
 pub mod tree;
 pub mod vfs;
+pub mod logging;

orchid-api/src/logging.rs

@@ -0,0 +1,13 @@
+use orchid_api_derive::{Coding, Hierarchy};
+
+use crate::proto::ExtHostNotif;
+
+#[derive(Clone, Debug, Coding)]
+pub enum LogStrategy {
+  StdErr,
+  File(String)
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostNotif)]
+pub struct Log(pub String);

orchid-api/src/proto.rs

@@ -27,18 +27,23 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::{atom, error, expr, interner, parser, system, tree, vfs};
+use crate::{atom, error, expr, interner, logging::{self, LogStrategy}, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
-pub struct HostHeader;
+pub struct HostHeader {
+  pub log_strategy: LogStrategy,
+}
 impl Decode for HostHeader {
   fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, HOST_INTRO);
-    Self
+    Self { log_strategy: LogStrategy::decode(read) }
   }
 }
 impl Encode for HostHeader {
-  fn encode<W: Write + ?Sized>(&self, write: &mut W) { write_exact(write, HOST_INTRO) }
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+    write_exact(write, HOST_INTRO);
+    self.log_strategy.encode(write)
+  }
 }
 
 static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
@@ -83,6 +88,7 @@ pub enum ExtHostReq {
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
   AdviseSweep(interner::AdviseSweep),
+  Log(logging::Log),
 }
 
 pub struct ExtHostChannel;
@@ -134,3 +140,38 @@ impl MsgSet for HostMsgSet {
   type In = ExtHostChannel;
   type Out = HostExtChannel;
 }
+
+#[cfg(test)]
+mod tests {
+    use ordered_float::NotNan;
+    use system::{SysDeclId, SystemDecl};
+
+    use super::*;
+
+    #[test]
+    fn host_header_enc() {
+      let hh = HostHeader { log_strategy: LogStrategy::File("SomeFile".to_string()) };
+      let mut enc = &hh.enc_vec()[..];
+      eprintln!("Encoded to {enc:?}");
+      HostHeader::decode(&mut enc);
+      assert_eq!(enc, []);
+    }
+
+    #[test]
+    fn ext_header_enc() {
+      let eh = ExtensionHeader {
+        systems: vec![
+          SystemDecl {
+            id: SysDeclId(1.try_into().unwrap()),
+            name: "misc".to_string(),
+            depends: vec![ "std".to_string() ],
+            priority: NotNan::new(1f64).unwrap()
+          }
+        ]
+      };
+      let mut enc = &eh.enc_vec()[..];
+      eprintln!("Encoded to {enc:?}");
+      ExtensionHeader::decode(&mut enc);
+      assert_eq!(enc, [])
+    }
+}

orchid-api/src/tree.rs

@@ -64,7 +64,7 @@ pub struct Placeholder {
 pub enum PlaceholderKind {
   Scalar,
   Name,
-  Vector { nonzero: bool, priority: u8 },
+  Vector { nz: bool, prio: u8 },
 }
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]

orchid-base/src/char_filter.rs

@@ -1,11 +1,11 @@
-use std::ops::RangeInclusive;
+use std::{fmt, ops::RangeInclusive};
 
 use itertools::Itertools;
 use orchid_api::parser::CharFilter;
 
 pub type CRange = RangeInclusive<char>;
 
-pub trait ICFilter {
+pub trait ICFilter: fmt::Debug {
   fn ranges(&self) -> &[RangeInclusive<char>];
 }
 impl ICFilter for [RangeInclusive<char>] {
@@ -27,7 +27,7 @@ fn try_merge_char_ranges(left: CRange, right: CRange) -> Result<CRange, (CRange,
 pub fn mk_char_filter(items: impl IntoIterator<Item = CRange>) -> CharFilter {
   CharFilter(
     (items.into_iter())
-      .filter(|r| *r.start() as u32 + 1 < *r.end() as u32)
+      .filter(|r| *r.start() as u32 <= *r.end() as u32)
       .sorted_by_key(|r| *r.start() as u32)
       .coalesce(try_merge_char_ranges)
       .collect_vec(),

orchid-base/src/id_store.rs

@@ -24,6 +24,10 @@ impl<T> IdStore<T> {
     let id64 = id.into();
     if tbl_g.contains_key(&id64) { Some(IdRecord(id64, tbl_g)) } else { None }
   }
+  pub fn is_empty(&self) -> bool { self.len() == 0 }
+  pub fn len(&self) -> usize {
+    self.table.get().map(|t| t.lock().unwrap().len()).unwrap_or(0)
+  }
 }
 
 impl<T> Default for IdStore<T> {

orchid-base/src/interner.rs

@@ -315,11 +315,30 @@ pub fn sweep_master(retained: Retained) {
   g.interners.vecs.sweep_master(retained.vecs.into_iter().collect());
 }
 
-#[test]
+#[cfg(test)]
-fn test_i() {
+mod test {
-  let _: Tok<String> = intern!(str: "foo");
+  use super::*;
-  let _: Tok<Vec<Tok<String>>> = intern!([Tok<String>]: &[
+
-    intern!(str: "bar"),
+    use std::num::NonZero;
-    intern!(str: "baz")
+
-  ]);
+    use orchid_api::interner::TStr;
+    use orchid_api_traits::{Decode, Encode};
+
+  #[test]
+  fn test_i() {
+    let _: Tok<String> = intern!(str: "foo");
+    let _: Tok<Vec<Tok<String>>> = intern!([Tok<String>]: &[
+      intern!(str: "bar"),
+      intern!(str: "baz")
+    ]);
+  }
+
+  #[test]
+  fn test_coding() {
+    let coded = TStr(NonZero::new(3u64).unwrap());
+    let mut enc = &coded.enc_vec()[..];
+    eprintln!("Coded {enc:?}");
+    TStr::decode(&mut enc);
+    assert_eq!(enc, [])
+  }
 }

orchid-base/src/lib.rs

@@ -18,3 +18,4 @@ pub mod reqnot;
 pub mod sequence;
 pub mod tokens;
 pub mod tree;
+pub mod logging;

orchid-base/src/logging.rs

@@ -0,0 +1,16 @@
+use std::{fs::File, io::Write};
+
+pub use orchid_api::logging::LogStrategy;
+
+pub struct Logger(LogStrategy);
+impl Logger {
+  pub fn new(strat: LogStrategy) -> Self { Self(strat) }
+  pub fn log(&self, msg: String) {
+    match &self.0 {
+      LogStrategy::StdErr => eprintln!("{msg}"),
+      LogStrategy::File(f) => writeln!(File::open(f).unwrap(), "{msg}").unwrap(),
+    }
+  }
+  pub fn strat(&self) -> LogStrategy { self.0.clone() }
+}
+

orchid-base/src/msg.rs

@@ -1,15 +1,16 @@
+use orchid_api_traits::Decode;
 use std::io;
 
+use orchid_api_traits::Encode;
+
 pub fn send_msg(write: &mut impl io::Write, msg: &[u8]) -> io::Result<()> {
-  write.write_all(&(u32::try_from(msg.len()).unwrap()).to_be_bytes())?;
+  u32::try_from(msg.len()).unwrap().encode(write);
   write.write_all(msg)?;
   write.flush()
 }
 
 pub fn recv_msg(read: &mut impl io::Read) -> io::Result<Vec<u8>> {
-  let mut len = [0u8; 4];
+  let len = u32::decode(read);
-  read.read_exact(&mut len)?;
-  let len = u32::from_be_bytes(len);
   let mut msg = vec![0u8; len as usize];
   read.read_exact(&mut msg)?;
   Ok(msg)

orchid-base/src/reqnot.rs

@@ -1,5 +1,5 @@
 use std::marker::PhantomData;
-use std::mem;
+use std::{mem, thread};
 use std::ops::{BitAnd, Deref};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{sync_channel, SyncSender};
@@ -12,7 +12,7 @@ use trait_set::trait_set;
 
 trait_set! {
   pub trait SendFn<T: MsgSet> = for<'a> FnMut(&'a [u8], ReqNot<T>) + DynClone + Send + 'static;
-  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + DynClone + Send + 'static;
+  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + DynClone + Send + Sync + 'static;
   pub trait NotifFn<T: MsgSet> =
     for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + DynClone + Send + Sync + 'static;
 }
@@ -31,7 +31,7 @@ impl<MS: MsgSet + 'static> RequestHandle<MS> {
   pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
   pub fn req(&self) -> &<MS::In as Channel>::Req { &self.message }
   fn respond(&self, response: &impl Encode) {
-    assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded");
+    assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id);
     let mut buf = (!self.id).to_be_bytes().to_vec();
     response.encode(&mut buf);
     let mut send = clone_box(&*self.reqnot().0.lock().unwrap().send);
@@ -45,7 +45,8 @@ impl<MS: MsgSet + 'static> RequestHandle<MS> {
 }
 impl<MS: MsgSet> Drop for RequestHandle<MS> {
   fn drop(&mut self) {
-    debug_assert!(self.fulfilled.load(Ordering::Relaxed), "Request dropped without response")
+    let done = self.fulfilled.load(Ordering::Relaxed);
+    debug_assert!(done, "Request {} dropped without response", self.id)
   }
 }
 
@@ -63,6 +64,9 @@ pub struct ReqNotData<T: MsgSet> {
   responses: HashMap<u64, SyncSender<Vec<u8>>>,
 }
 
+/// Wraps a raw message buffer to save on copying.
+/// Dereferences to the tail of the message buffer, cutting off the ID
+#[derive(Debug, Clone)]
 pub struct RawReply(Vec<u8>);
 impl Deref for RawReply {
   type Target = [u8];
@@ -96,7 +100,8 @@ impl<T: MsgSet> ReqNot<T> {
       let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
       let mut req = clone_box(&*g.req);
       mem::drop(g);
-      req(RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() })
+      let handle = RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() };
+      thread::Builder::new().name(format!("request {id}")).spawn(move || req(handle)).unwrap();
     }
   }
 
@@ -109,6 +114,12 @@ impl<T: MsgSet> ReqNot<T> {
   }
 }
 
+pub trait DynRequester: Send + Sync {
+  type Transfer;
+  /// Encode and send a request, then receive the response buffer.
+  fn raw_request(&self, data: Self::Transfer) -> RawReply;
+}
+
 pub struct MappedRequester<'a, T>(Box<dyn Fn(T) -> RawReply + Send + Sync + 'a>);
 impl<'a, T> MappedRequester<'a, T> {
   fn new<U: DynRequester + 'a>(req: U) -> Self
@@ -139,10 +150,6 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
   }
 }
 
-pub trait DynRequester: Send + Sync {
-  type Transfer;
-  fn raw_request(&self, data: Self::Transfer) -> RawReply;
-}
 pub trait Requester: DynRequester {
   #[must_use = "These types are subject to change with protocol versions. \
     If you don't want to use the return value, At a minimum, force the type."]

orchid-base/src/tokens.rs

@@ -1,7 +1,12 @@
-use orchid_api::tree::{Placeholder, PlaceholderKind};
+use std::fmt::Display;
+
+use orchid_api::tree::{Paren, Placeholder, PlaceholderKind};
 
 use crate::interner::{deintern, Tok};
 
+pub const PARENS: &[(char, char, Paren)] =
+  &[('(', ')', Paren::Round), ('[', ']', Paren::Square), ('{', '}', Paren::Curly)];
+
 #[derive(Clone, Debug)]
 pub struct OwnedPh {
   pub name: Tok<String>,
@@ -13,3 +18,16 @@ impl OwnedPh {
   }
   pub fn from_api(ph: Placeholder) -> Self { Self { name: deintern(ph.name), kind: ph.kind } }
 }
+
+impl Display for OwnedPh {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    match self.kind {
+      PlaceholderKind::Name => write!(f, "$_{}", self.name),
+      PlaceholderKind::Scalar => write!(f, "${}", self.name),
+      PlaceholderKind::Vector { nz: false, prio: 0 } => write!(f, "..${}", self.name),
+      PlaceholderKind::Vector { nz: true, prio: 0 } => write!(f, "...${}", self.name),
+      PlaceholderKind::Vector { nz: false, prio } => write!(f, "..${}:{prio}", self.name),
+      PlaceholderKind::Vector { nz: true, prio } => write!(f, "...${}:{prio}", self.name),
+    }
+  }
+}

orchid-extension/src/atom.rs

@@ -108,6 +108,7 @@ pub trait AtomDynfo: Send + Sync + 'static {
   fn call(&self, ctx: AtomCtx<'_>, arg: ExprTicket) -> GenExpr;
   fn call_ref(&self, ctx: AtomCtx<'_>, arg: ExprTicket) -> GenExpr;
   fn same(&self, ctx: AtomCtx<'_>, buf2: &[u8]) -> bool;
+  fn print(&self, ctx: AtomCtx<'_>) -> String;
   fn handle_req(&self, ctx: AtomCtx<'_>, req: &mut dyn Read, rep: &mut dyn Write);
   fn command(&self, ctx: AtomCtx<'_>) -> ProjectResult<Option<GenExpr>>;
   fn drop(&self, ctx: AtomCtx<'_>);
@@ -127,8 +128,8 @@ impl Clone for AtomFactory {
   fn clone(&self) -> Self { AtomFactory(clone_box(&*self.0)) }
 }
 
-pub struct ErrorNotCallable;
+pub struct ErrNotCallable;
-impl ProjectError for ErrorNotCallable {
+impl ProjectError for ErrNotCallable {
   const DESCRIPTION: &'static str = "This atom is not callable";
 }
 

orchid-extension/src/atom_owned.rs

@@ -11,12 +11,11 @@ use orchid_api_traits::{Decode, Encode};
 use orchid_base::id_store::{IdRecord, IdStore};
 
 use crate::atom::{
-  AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant,
+  get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, ErrNotCallable, ErrorNotCommand, ReqPck, RequestPack
-  ErrorNotCallable, ErrorNotCommand, ReqPck, RequestPack,
 };
 use crate::error::ProjectResult;
 use crate::expr::{bot, ExprHandle, GenExpr};
-use crate::system::{atom_info_for, SysCtx};
+use crate::system::SysCtx;
 
 pub struct OwnedVariant;
 impl AtomicVariant for OwnedVariant {}
@@ -24,7 +23,7 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
   fn _factory(self) -> AtomFactory {
     AtomFactory::new(move |sys| {
       let rec = OBJ_STORE.add(Box::new(self));
-      let mut data = atom_info_for(sys.dyn_card(), rec.atom_tid()).expect("obj exists").0.enc_vec();
+      let mut data = get_info::<A>(sys.dyn_card()).0.enc_vec();
       rec.id().encode(&mut data);
       rec.encode(&mut data);
       LocalAtom { drop: true, data }
@@ -35,11 +34,15 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 }
 
 fn with_atom<U>(mut b: &[u8], f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>) -> U) -> U {
-  f(OBJ_STORE.get(NonZeroU64::decode(&mut b)).expect("Received invalid atom ID"))
+  let id = NonZeroU64::decode(&mut b);
+  f(OBJ_STORE.get(id).unwrap_or_else(|| panic!("Received invalid atom ID: {id}")))
 }
 
 pub struct OwnedAtomDynfo<T: OwnedAtom>(PhantomData<T>);
 impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
+  fn print(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> String {
+    with_atom(buf, |a| a.dyn_print(ctx))
+  }
   fn tid(&self) -> TypeId { TypeId::of::<T>() }
   fn name(&self) -> &'static str { type_name::<T>() }
   fn decode(&self, AtomCtx(data, _): AtomCtx) -> Box<dyn Any> {
@@ -67,7 +70,7 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone + 'static {
   fn val(&self) -> Cow<'_, Self::Data>;
   #[allow(unused_variables)]
-  fn call_ref(&self, arg: ExprHandle) -> GenExpr { bot(ErrorNotCallable) }
+  fn call_ref(&self, arg: ExprHandle) -> GenExpr { bot(ErrNotCallable) }
   fn call(self, arg: ExprHandle) -> GenExpr {
     let ctx = arg.get_ctx();
     let gcl = self.call_ref(arg);
@@ -87,6 +90,8 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone
   fn command(self, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> { Err(Arc::new(ErrorNotCommand)) }
   #[allow(unused_variables)]
   fn free(self, ctx: SysCtx) {}
+  #[allow(unused_variables)]
+  fn print(&self, ctx: SysCtx) -> String { format!("OwnedAtom({})", type_name::<Self>()) }
 }
 pub trait DynOwnedAtom: Send + Sync + 'static {
   fn atom_tid(&self) -> TypeId;
@@ -98,6 +103,7 @@ pub trait DynOwnedAtom: Send + Sync + 'static {
   fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write);
   fn dyn_command(self: Box<Self>, ctx: SysCtx) -> ProjectResult<Option<GenExpr>>;
   fn dyn_free(self: Box<Self>, ctx: SysCtx);
+  fn dyn_print(&self, ctx: SysCtx) -> String;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
   fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
@@ -123,6 +129,7 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
     self.command(ctx)
   }
   fn dyn_free(self: Box<Self>, ctx: SysCtx) { self.free(ctx) }
+  fn dyn_print(&self, ctx: SysCtx) -> String { self.print(ctx) }
 }
 
 pub(crate) static OBJ_STORE: IdStore<Box<dyn DynOwnedAtom>> = IdStore::new();

orchid-extension/src/atom_thin.rs

@@ -1,5 +1,4 @@
 use std::any::{type_name, Any, TypeId};
-use std::fmt;
 use std::io::Write;
 use std::marker::PhantomData;
 use std::sync::Arc;
@@ -10,7 +9,7 @@ use orchid_api_traits::{Coding, Decode, Encode};
 
 use crate::atom::{
   get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant,
-  ErrorNotCallable, ReqPck, RequestPack,
+  ErrNotCallable, ReqPck, RequestPack,
 };
 use crate::error::ProjectResult;
 use crate::expr::{bot, ExprHandle, GenExpr};
@@ -32,11 +31,10 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
 
 pub struct ThinAtomDynfo<T: ThinAtom>(PhantomData<T>);
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
+  fn print(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> String { T::decode(&mut &buf[..]).print(ctx) }
   fn tid(&self) -> TypeId { TypeId::of::<T>() }
   fn name(&self) -> &'static str { type_name::<T>() }
-  fn decode(&self, AtomCtx(data, _): AtomCtx) -> Box<dyn Any> {
+  fn decode(&self, AtomCtx(buf, _): AtomCtx) -> Box<dyn Any> { Box::new(T::decode(&mut &buf[..])) }
-    Box::new(T::decode(&mut &data[..]))
-  }
   fn call(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
     T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg))
   }
@@ -57,25 +55,24 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
   fn command(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> ProjectResult<Option<GenExpr>> {
     T::decode(&mut &buf[..]).command(ctx)
   }
-  fn drop(&self, AtomCtx(buf, _): AtomCtx) {
+  fn drop(&self, AtomCtx(buf, ctx): AtomCtx) {
-    eprintln!("Received drop signal for non-drop atom {:?}", T::decode(&mut &buf[..]))
+    let string_self = T::decode(&mut &buf[..]).print(ctx);
+    eprintln!("Received drop signal for non-drop atom {string_self:?}")
   }
 }
 
-pub trait ThinAtom: AtomCard<Data = Self> + Coding + fmt::Debug + Send + Sync + 'static {
+pub trait ThinAtom: AtomCard<Data = Self> + Coding + Send + Sync + 'static {
   #[allow(unused_variables)]
-  fn call(&self, arg: ExprHandle) -> GenExpr { bot(ErrorNotCallable) }
+  fn call(&self, arg: ExprHandle) -> GenExpr { bot(ErrNotCallable) }
   #[allow(unused_variables)]
   fn same(&self, ctx: SysCtx, other: &Self) -> bool {
-    eprintln!(
+    let tname = type_name::<Self>();
-      "Override ThinAtom::same for {} if it can be generated during parsing",
+    eprintln!("Override ThinAtom::same for {tname} if it can be generated during parsing");
-      type_name::<Self>()
-    );
     false
   }
   fn handle_req(&self, ctx: SysCtx, pck: impl ReqPck<Self>);
   #[allow(unused_variables)]
-  fn command(&self, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> {
+  fn command(&self, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> { Err(Arc::new(ErrNotCallable)) }
-    Err(Arc::new(ErrorNotCallable))
+  #[allow(unused_variables)]
-  }
+  fn print(&self, ctx: SysCtx) -> String { format!("ThinAtom({})", type_name::<Self>()) }
 }

orchid-extension/src/entrypoint.rs

@@ -1,12 +1,13 @@
+use std::io::Write;
 use std::num::NonZero;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
-use std::{mem, thread};
+use std::{mem, process, thread};
 
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_api::atom::{
-  Atom, AtomDrop, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwded, NextStep,
+  Atom, AtomDrop, AtomPrint, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwded, NextStep
 };
 use orchid_api::interner::Sweep;
 use orchid_api::parser::{CharFilter, LexExpr, LexedExpr, ParserReq};
@@ -18,6 +19,7 @@ use orchid_api_traits::{Decode, Encode};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::clone;
 use orchid_base::interner::{deintern, init_replica, sweep_replica};
+use orchid_base::logging::Logger;
 use orchid_base::name::PathSlice;
 use orchid_base::reqnot::{ReqNot, Requester};
 
@@ -31,8 +33,17 @@ use crate::system_ctor::{CtedObj, DynSystemCtor};
 use crate::tree::{LazyMemberFactory, TIACtxImpl};
 
 pub struct ExtensionData {
+  pub thread_name: &'static str,
   pub systems: &'static [&'static dyn DynSystemCtor],
 }
+impl ExtensionData {
+  pub fn new(thread_name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
+    Self { thread_name, systems }
+  }
+  pub fn main(self) {
+    extension_main(self)
+  }
+}
 
 pub enum MemberRecord {
   Gen(LazyMemberFactory),
@@ -60,7 +71,13 @@ pub fn with_atom_record<T>(
 }
 
 pub fn extension_main(data: ExtensionData) {
-  HostHeader::decode(&mut &recv_parent_msg().unwrap()[..]);
+  if thread::Builder::new().name(data.thread_name.to_string()).spawn(|| extension_main_logic(data)).unwrap().join().is_err() {
+    process::exit(-1)
+  }
+}
+
+fn extension_main_logic(data: ExtensionData) {
+  let HostHeader{ log_strategy } = HostHeader::decode(&mut std::io::stdin().lock());
   let mut buf = Vec::new();
   let decls = (data.systems.iter().enumerate())
     .map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
@@ -68,21 +85,26 @@ pub fn extension_main(data: ExtensionData) {
     .collect_vec();
   let systems = Arc::new(Mutex::new(HashMap::<SysId, SystemRecord>::new()));
   ExtensionHeader { systems: decls.clone() }.encode(&mut buf);
-  send_parent_msg(&buf).unwrap();
+  std::io::stdout().write_all(&buf).unwrap();
+  std::io::stdout().flush().unwrap();
   let exiting = Arc::new(AtomicBool::new(false));
+  let logger = Arc::new(Logger::new(log_strategy));
   let rn = ReqNot::<ExtMsgSet>::new(
-    |a, _| send_parent_msg(a).unwrap(),
+    |a, _| {
-    clone!(systems, exiting; move |n, reqnot| match n {
+      eprintln!("Upsending {:?}", a);
+      send_parent_msg(a).unwrap()
+    },
+    clone!(systems, exiting, logger; move |n, reqnot| match n {
       HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
       HostExtNotif::SystemDrop(SystemDrop(sys_id)) =>
         mem::drop(systems.lock().unwrap().remove(&sys_id)),
       HostExtNotif::AtomDrop(AtomDrop(atom)) => {
         with_atom_record(&systems, &atom, |rec, cted, data| {
-          rec.drop(AtomCtx(data, SysCtx{ reqnot, id: atom.owner, cted }))
+          rec.drop(AtomCtx(data, SysCtx{ reqnot, logger: logger.clone(), id: atom.owner, cted }))
         })
       }
     }),
-    clone!(systems; move |req| match req.req() {
+    clone!(systems, logger; move |req| match req.req() {
       HostExtReq::Ping(ping@Ping) => req.handle(ping, &()),
       HostExtReq::Sweep(sweep@Sweep) => req.handle(sweep, &sweep_replica()),
       HostExtReq::NewSystem(new_sys) => {
@@ -90,7 +112,8 @@ pub fn extension_main(data: ExtensionData) {
         let cted = data.systems[i].new_system(new_sys);
         let mut vfses = HashMap::new();
         let lex_filter = cted.inst().dyn_lexers().iter().fold(CharFilter(vec![]), |cf, lx| {
-          char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
+          let lxcf = mk_char_filter(lx.char_filter().iter().cloned());
+          char_filter_union(&cf, &lxcf)
         });
         let mut lazy_mems = HashMap::new();
         let const_root = (cted.inst().dyn_env().into_iter())
@@ -141,8 +164,8 @@ pub fn extension_main(data: ExtensionData) {
         let tk = req.will_handle_as(lex);
         thread::spawn(clone!(systems; move || {
           let ctx = LexContext { sys, id, pos, reqnot: req.reqnot(), text: &text };
-          let first_char = text.chars().next().unwrap();
+          let trigger_char = text.chars().nth(pos as usize).unwrap();
-          for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), first_char)) {
+          for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char)) {
             match lx.lex(&text[pos as usize..], &ctx) {
               Err(e) if e.as_any_ref().is::<NotApplicableLexerError>() => continue,
               Err(e) if e.as_any_ref().is::<CascadingError>() => return req.handle_as(tk, &None),
@@ -155,6 +178,7 @@ pub fn extension_main(data: ExtensionData) {
               }
             }
           }
+          eprintln!("Got notified about n/a character '{trigger_char}'");
           req.handle_as(tk, &None)
         }));
       },
@@ -162,10 +186,16 @@ pub fn extension_main(data: ExtensionData) {
         let systems_g = systems.lock().unwrap();
         let atom = atom_req.get_atom();
         let sys = &systems_g[&atom.owner];
-        let ctx = SysCtx { cted: sys.cted.clone(), id: atom.owner, reqnot: req.reqnot() };
+        let ctx = SysCtx {
+          cted: sys.cted.clone(),
+          id: atom.owner,
+          logger: logger.clone(),
+          reqnot: req.reqnot()
+        };
         let dynfo = resolv_atom(&*sys.cted.inst(), atom);
         let actx = AtomCtx(&atom.data[8..], ctx);
         match atom_req {
+          AtomReq::AtomPrint(print@AtomPrint(_)) => req.handle(print, &dynfo.print(actx)),
           AtomReq::AtomSame(same@AtomSame(_, r)) => {
             // different systems or different type tags
             if atom.owner != r.owner || atom.data[..8] != r.data[..8] {
@@ -195,6 +225,8 @@ pub fn extension_main(data: ExtensionData) {
   );
   init_replica(rn.clone().map());
   while !exiting.load(Ordering::Relaxed) {
-    rn.receive(recv_parent_msg().unwrap())
+    let rcvd = recv_parent_msg().unwrap();
+    // eprintln!("Downsent {rcvd:?}");
+    rn.receive(rcvd)
   }
 }

orchid-extension/src/system.rs

@@ -1,4 +1,5 @@
 use std::any::TypeId;
+use std::sync::Arc;
 
 use hashbrown::HashMap;
 use orchid_api::atom::Atom;
@@ -6,6 +7,7 @@ use orchid_api::proto::ExtMsgSet;
 use orchid_api::system::SysId;
 use orchid_api_traits::Decode;
 use orchid_base::interner::Tok;
+use orchid_base::logging::Logger;
 use orchid_base::reqnot::ReqNot;
 
 use crate::atom::{get_info, AtomCtx, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom};
@@ -106,4 +108,5 @@ pub struct SysCtx {
   pub reqnot: ReqNot<ExtMsgSet>,
   pub id: SysId,
   pub cted: CtedObj,
+  pub logger: Arc<Logger>,
 }

orchid-extension/src/tree.rs

@@ -47,7 +47,7 @@ pub fn ph(s: &str) -> OwnedPh {
       if konst::string::starts_with(name, "_") {
         panic!("Names starting with an underscore indicate a single-name scalar placeholder")
       }
-      OwnedPh { name: intern(name), kind: PlaceholderKind::Vector { nonzero, priority } }
+      OwnedPh { name: intern(name), kind: PlaceholderKind::Vector { nz: nonzero, prio: priority } }
     },
     None => match konst::string::strip_prefix(s, "$_") {
       Some(name) => OwnedPh { name: intern(name), kind: PlaceholderKind::Name },

orchid-host/Cargo.toml

@@ -15,3 +15,4 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "4.2.0"
+substack = "1.1.0"

orchid-host/src/child.rs

@@ -1,5 +1,5 @@
 use std::sync::Mutex;
-use std::{io, mem, process};
+use std::{fmt, io, mem, process};
 
 use orchid_base::msg::{recv_msg, send_msg};
 
@@ -7,20 +7,33 @@ pub struct SharedChild {
   child: process::Child,
   stdin: Mutex<process::ChildStdin>,
   stdout: Mutex<process::ChildStdout>,
+  debug: Option<(String, Mutex<Box<dyn fmt::Write>>)>,
 }
 impl SharedChild {
-  pub fn new(cmd: &mut process::Command) -> io::Result<Self> {
+  pub fn new(command: &mut process::Command, debug: Option<(&str, impl fmt::Write + 'static)>) -> io::Result<Self> {
-    let mut child = cmd.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
+    let mut child = command.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
     let stdin = Mutex::new(child.stdin.take().expect("Piped stdin above"));
     let stdout = Mutex::new(child.stdout.take().expect("Piped stdout above"));
-    Ok(Self { stdin, stdout, child })
+    let debug = debug.map(|(n, w)| (n.to_string(), Mutex::new(Box::new(w) as Box<dyn fmt::Write>)));
+    Ok(Self { child, stdin, stdout, debug })
   }
 
   pub fn send_msg(&self, msg: &[u8]) -> io::Result<()> {
+    if let Some((n, dbg)) = &self.debug {
+      let mut dbg = dbg.lock().unwrap();
+      writeln!(dbg, "To {n}: {msg:?}").unwrap();
+    }
     send_msg(&mut *self.stdin.lock().unwrap(), msg)
   }
 
-  pub fn recv_msg(&self) -> io::Result<Vec<u8>> { recv_msg(&mut *self.stdout.lock().unwrap()) }
+  pub fn recv_msg(&self) -> io::Result<Vec<u8>> {
+    let msg = recv_msg(&mut *self.stdout.lock().unwrap());
+    if let Some((n, dbg)) = &self.debug {
+      let mut dbg = dbg.lock().unwrap();
+      writeln!(dbg, "From {n}: {msg:?}").unwrap();
+    }
+    msg
+  }
 }
 impl Drop for SharedChild {
   fn drop(&mut self) { mem::drop(self.child.kill()) }

orchid-host/src/expr.rs

@@ -29,7 +29,9 @@ impl RtExpr {
     self.id()
   }
   pub fn resolve(tk: ExprTicket) -> Option<Self> { KNOWN_EXPRS.read().unwrap().get(&tk).cloned() }
-  pub fn from_api(api: Expr, sys: &System) -> Self { todo!() }
+  pub fn from_api(api: Expr, sys: &System) -> Self {
+    Self { data: Arc::default(), is_canonical: Arc::default() }
+  }
 }
 impl Drop for RtExpr {
   fn drop(&mut self) {

orchid-host/src/extension.rs

@@ -1,6 +1,13 @@
-use std::io::Write as _;
+use orchid_api::logging::Log;
+use orchid_base::logging::Logger;
+use orchid_base::msg::{recv_msg, send_msg};
+use substack::{Stackframe, Substack};
+use std::collections::VecDeque;
+use std::io::{stderr, BufRead, BufReader, Write as _};
 use std::num::NonZero;
 use std::ops::Deref;
+use std::path::PathBuf;
+use std::process::ChildStdin;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
 use std::sync::mpsc::{sync_channel, SyncSender};
 use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
@@ -11,7 +18,7 @@ use hashbrown::hash_map::Entry;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use lazy_static::lazy_static;
-use orchid_api::atom::{Atom, AtomDrop, AtomSame, CallRef, FinalCall, Fwd, Fwded};
+use orchid_api::atom::{Atom, AtomDrop, AtomPrint, AtomSame, CallRef, FinalCall, Fwd, Fwded};
 use orchid_api::error::ProjResult;
 use orchid_api::expr::{Acquire, Expr, ExprNotif, ExprTicket, Release, Relocate};
 use orchid_api::interner::IntReq;
@@ -83,6 +90,7 @@ impl AtomHand {
     self.0.owner.reqnot().request(Fwded(self.0.api_ref(), req))
   }
   pub fn api_ref(&self) -> Atom { self.0.api_ref() }
+  pub fn print(&self) -> String { self.0.owner.reqnot().request(AtomPrint(self.0.api_ref())) }
 }
 
 /// Data held about an Extension. This is refcounted within [Extension]. It's
@@ -92,11 +100,16 @@ impl AtomHand {
 #[derive(destructure)]
 pub struct ExtensionData {
   child: Mutex<process::Child>,
+  child_stdin: Mutex<ChildStdin>,
   reqnot: ReqNot<HostMsgSet>,
   systems: Vec<SystemCtor>,
+  logger: Logger,
 }
 impl Drop for ExtensionData {
-  fn drop(&mut self) { self.reqnot.notify(HostExtNotif::Exit) }
+  fn drop(&mut self) {
+    self.reqnot.notify(HostExtNotif::Exit);
+    self.child.lock().unwrap().wait().expect("Extension exited with error");
+  }
 }
 
 fn acq_expr(sys: SysId, extk: ExprTicket) {
@@ -115,27 +128,46 @@ fn rel_expr(sys: SysId, extk: ExprTicket) {
 #[derive(Clone)]
 pub struct Extension(Arc<ExtensionData>);
 impl Extension {
-  pub fn new(mut cmd: process::Command) -> io::Result<Self> {
+  pub fn new(mut cmd: process::Command, logger: Logger) -> io::Result<Self> {
-    let mut child = cmd.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
+    let mut child = cmd
-    HostHeader.encode(child.stdin.as_mut().unwrap());
+      .stdin(process::Stdio::piped())
-    let eh = ExtensionHeader::decode(child.stdout.as_mut().unwrap());
+      .stdout(process::Stdio::piped())
-    Ok(Self(Arc::new_cyclic(|weak| ExtensionData {
+      .stderr(process::Stdio::piped())
+      .spawn()?;
+    let mut child_stdin = child.stdin.take().unwrap();
+    let mut child_stdout = child.stdout.take().unwrap();
+    let child_stderr = child.stderr.take().unwrap();
+    thread::Builder::new().name("stderr forwarder".to_string()).spawn(|| {
+      let mut reader = BufReader::new(child_stderr);
+      loop {
+        let mut buf = String::new();
+        if 0 == reader.read_line(&mut buf).unwrap() {
+          break;
+        }
+        stderr().write_all(buf.as_bytes()).unwrap();
+      }
+    }).unwrap();
+    HostHeader{ log_strategy: logger.strat() }.encode(&mut child_stdin);
+    let eh = ExtensionHeader::decode(&mut child_stdout);
+    let ret = Arc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
+      logger,
       child: Mutex::new(child),
+      child_stdin: Mutex::new(child_stdin),
       reqnot: ReqNot::new(
         clone!(weak; move |sfn, _| {
-          let arc: Arc<ExtensionData> = weak.upgrade().unwrap();
+          eprintln!("Downsending {:?}", sfn);
-          let mut g = arc.child.lock().unwrap();
+          send_msg(&mut *weak.upgrade().unwrap().child_stdin.lock().unwrap(), sfn).unwrap();
-          g.stdin.as_mut().unwrap().write_all(sfn).unwrap();
         }),
-        |notif, _| match notif {
+        clone!(weak; move |notif, _| match notif {
           ExtHostNotif::ExprNotif(ExprNotif::Acquire(Acquire(sys, extk))) => acq_expr(sys, extk),
           ExtHostNotif::ExprNotif(ExprNotif::Release(Release(sys, extk))) => rel_expr(sys, extk),
           ExtHostNotif::ExprNotif(ExprNotif::Relocate(Relocate { dec, inc, expr })) => {
             acq_expr(inc, expr);
             rel_expr(dec, expr);
           },
-          ExtHostNotif::AdviseSweep(_advice) => eprintln!("Sweep advice is unsupported")
+          ExtHostNotif::AdviseSweep(_advice) => eprintln!("Sweep advice is unsupported"),
-        },
+          ExtHostNotif::Log(Log(str)) => weak.upgrade().unwrap().logger.log(str),
+        }),
         |req| match req.req() {
           ExtHostReq::Ping(ping) => req.handle(ping, &()),
           ExtHostReq::IntReq(IntReq::InternStr(s)) => req.handle(s, &intern(&**s.0).marker()),
@@ -160,7 +192,19 @@ impl Extension {
         },
       ),
       systems: eh.systems.into_iter().map(|decl| SystemCtor { decl, ext: weak.clone() }).collect(),
-    })))
+    });
+    let weak = Arc::downgrade(&ret);
+    let prog_pbuf = PathBuf::from(cmd.get_program());
+    let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy();
+    thread::Builder::new().name(format!("host-end:{}", prog)).spawn(move || {
+      loop {
+        let ingress = recv_msg(&mut child_stdout).expect("could not receive");
+        if let Some(sys) = weak.upgrade() {
+          sys.reqnot.receive(ingress);
+        }
+      }
+    }).unwrap();
+    Ok(Self(ret))
   }
   pub fn systems(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
 }
@@ -293,3 +337,54 @@ impl Deref for System {
   type Target = SystemInstData;
   fn deref(&self) -> &Self::Target { self.0.as_ref() }
 }
+
+#[derive(Debug, Clone)]
+pub enum SysResolvErr {
+  Loop(Vec<String>),
+  Missing(String)
+}
+
+pub fn init_systems(tgts: &[String], exts: &[Extension]) -> Result<Vec<System>, SysResolvErr> {
+  let mut to_load = HashMap::<&str, &SystemCtor>::new();
+  let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque::<&str>>();
+  while let Some(target) = to_find.pop_front() {
+    if to_load.contains_key(target) {
+      continue;
+    }
+    let ctor = (exts.iter())
+      .flat_map(|e| e.systems().filter(|c| c.decl.name == target))
+      .max_by_key(|c| c.decl.priority)
+      .ok_or_else(|| SysResolvErr::Missing(target.to_string()))?;
+    to_load.insert(target, ctor);
+    to_find.extend(ctor.decl.depends.iter().map(|s| s.as_str()));
+  }
+  let mut to_load_ordered = Vec::new();
+  fn walk_deps<'a>(
+    graph: &mut HashMap::<&str, &'a SystemCtor>,
+    list: &mut Vec<&'a SystemCtor>,
+    chain: Stackframe<&str>
+  ) -> Result<(), SysResolvErr> {
+    if let Some(ctor) = graph.remove(chain.item) {
+      // if the above is none, the system is already queued. Missing systems are detected above
+      for dep in ctor.decl.depends.iter() {
+        if Substack::Frame(chain).iter().any(|c| c == dep) {
+          let mut circle = vec![dep.to_string()];
+          circle.extend(Substack::Frame(chain).iter().map(|s| s.to_string()));
+          return Err(SysResolvErr::Loop(circle))
+        }
+        walk_deps(graph, list, Substack::Frame(chain).new_frame(dep))?
+      }
+      list.push(ctor);
+    }
+    Ok(())
+  }
+  for tgt in tgts {
+    walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
+  }
+  let mut systems = HashMap::<&str, System>::new();
+  for ctor in to_load_ordered.iter() {
+    let sys = ctor.run(ctor.depends().map(|n| &systems[n]));
+    systems.insert(ctor.name(), sys);
+  }
+  Ok(systems.into_values().collect_vec())
+}

orchid-host/src/lex.rs

@@ -3,12 +3,12 @@ use std::num::NonZeroU64;
 use hashbrown::HashMap;
 use orchid_api::parser::SubLexed;
 use orchid_api::system::SysId;
-use orchid_api::tree::{Paren, Token, TokenTree, TreeTicket};
+use orchid_api::tree::{Token, TokenTree, TreeTicket};
 use orchid_base::error::OwnedError;
 use orchid_base::intern;
 use orchid_base::interner::{deintern, intern, Tok};
 use orchid_base::location::Pos;
-use orchid_base::tokens::OwnedPh;
+use orchid_base::tokens::{OwnedPh, PARENS};
 
 use crate::extension::{AtomHand, System};
 use crate::results::{mk_err, OwnedResult};
@@ -69,9 +69,6 @@ impl<'a> LexCtx<'a> {
   }
 }
 
-const PARENS: &[(char, char, Paren)] =
-  &[('(', ')', Paren::Round), ('[', ']', Paren::Square), ('{', '}', Paren::Curly)];
-
 pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
   let start = ctx.get_pos();
   assert!(
@@ -164,7 +161,7 @@ pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
 
 fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
 fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
-fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}:\\".contains(c) }
+fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}\\".contains(c) }
 fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 
 fn tt_to_owned(api: &TokenTree, sys: SysId, ctx: &mut LexCtx<'_>) -> OwnedTokTree {

orchid-host/src/tree.rs

@@ -1,4 +1,6 @@
 use std::borrow::Borrow;
+use std::cell::RefCell;
+use std::fmt::{Display, Write};
 use std::ops::Range;
 use std::sync::{Mutex, OnceLock};
 
@@ -9,7 +11,7 @@ use orchid_base::error::OwnedError;
 use orchid_base::interner::{deintern, Tok};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use orchid_base::tokens::OwnedPh;
+use orchid_base::tokens::{OwnedPh, PARENS};
 use ordered_float::NotNan;
 
 use crate::expr::RtExpr;
@@ -49,6 +51,9 @@ impl OwnedTokTree {
     tokv.into_iter().map(|t| Self::from_api(t.borrow(), sys, do_slot)).collect()
   }
 }
+impl Display for OwnedTokTree {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
+}
 
 #[derive(Clone, Debug)]
 pub enum OwnedTok {
@@ -62,6 +67,52 @@ pub enum OwnedTok {
   Ph(OwnedPh),
   Bottom(Vec<OwnedError>),
 }
+impl Display for OwnedTok {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    thread_local! {
+      static PAREN_LEVEL: RefCell<usize> = 0.into();
+    }
+    fn get_indent() -> usize { PAREN_LEVEL.with_borrow(|t| *t) }
+    fn with_indent<T>(f: impl FnOnce() -> T) -> T {
+      PAREN_LEVEL.with_borrow_mut(|t| *t += 1);
+      let r = f();
+      PAREN_LEVEL.with_borrow_mut(|t| *t -= 1);
+      r
+    }
+    match self {
+      Self::Atom(ah) => f.write_str(&indent(&ah.print(), get_indent(), false)),
+      Self::BR => write!(f, "\n{}", "  ".repeat(get_indent())),
+      Self::Bottom(err) => write!(f, "Botttom({})",
+        err.iter().map(|e| format!("{}: {}", e.description, e.message)).join(", ")
+      ),
+      Self::Comment(c) => write!(f, "--[{c}]--"),
+      Self::Lambda(arg) => with_indent(|| write!(f, "\\ {} .", fmt_tt_v(arg))),
+      Self::NS => f.write_str("::"),
+      Self::Name(n) => f.write_str(n),
+      Self::Ph(ph) => write!(f, "{ph}"),
+      Self::S(p, b) => {
+        let (lp, rp, _) = PARENS.iter().find(|(_, _, par)| par == p).unwrap();
+        f.write_char(*lp)?;
+        with_indent(|| f.write_str(&fmt_tt_v(b)))?;
+        f.write_char(*rp)
+      }
+    }
+  }
+}
+
+pub fn fmt_tt_v<'a>(ttv: impl IntoIterator<Item = &'a OwnedTokTree>) -> String {
+  ttv.into_iter().join(" ")
+}
+
+pub fn indent(s: &str, lvl: usize, first: bool) -> String {
+  if first {
+    s.replace("\n", &("\n".to_string() + &"  ".repeat(lvl)))
+  } else if let Some((fst, rest)) = s.split_once('\n') {
+    fst.to_string() + "\n" + &indent(rest, lvl, true)
+  } else {
+    s.to_string()
+  }
+}
 
 #[derive(Debug)]
 pub struct OwnedItem {

orchid-std/src/main.rs

@@ -1,4 +1,4 @@
-use orchid_extension::entrypoint::{extension_main, ExtensionData};
+use orchid_extension::entrypoint::ExtensionData;
 use orchid_std::StdSystem;
 
-pub fn main() { extension_main(ExtensionData { systems: &[&StdSystem] }) }
+pub fn main() { ExtensionData::new("orchid-std::main", &[&StdSystem]).main() }

orchid-std/src/std.rs

@@ -9,7 +9,7 @@ use orchid_extension::system_ctor::SystemCtor;
 use orchid_extension::tree::{cnst, module, root_mod, GenMemberKind};
 
 use crate::number::num_atom::{Float, Int};
-use crate::string::str_atom::StrAtom;
+use crate::string::str_atom::{IntStrAtom, StrAtom};
 use crate::string::str_lexer::StringLexer;
 use crate::OrcString;
 
@@ -25,7 +25,7 @@ impl SystemCtor for StdSystem {
 impl SystemCard for StdSystem {
   type Ctor = Self;
   const ATOM_DEFS: &'static [Option<&'static dyn AtomDynfo>] =
-    &[Some(Int::INFO), Some(Float::INFO), Some(StrAtom::INFO)];
+    &[Some(Int::INFO), Some(Float::INFO), Some(StrAtom::INFO), Some(IntStrAtom::INFO)];
 }
 impl System for StdSystem {
   fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }

orchid-std/src/string/str_atom.rs

@@ -61,6 +61,7 @@ impl From<Tok<String>> for IntStrAtom {
 impl OwnedAtom for IntStrAtom {
   fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.marker()) }
   fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+  fn print(&self, _ctx: SysCtx) -> String { format!("{:?}i", self.0.as_str()) }
 }
 
 #[derive(Clone)]

orcx/Cargo.toml

@@ -6,3 +6,8 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+camino = "1.1.7"
+clap = { version = "=4.5.4", features = ["derive"] }
+itertools = "0.13.0"
+orchid-base = { version = "0.1.0", path = "../orchid-base" }
+orchid-host = { version = "0.1.0", path = "../orchid-host" }

orcx/src/main.rs

@@ -1,3 +1,43 @@
-fn main() {
+use std::{fs::File, io::Read, process::Command};
-  println!("Hello, world!");
+
+use camino::Utf8PathBuf;
+use clap::{Parser, Subcommand};
+use itertools::Itertools;
+use orchid_base::{interner::intern, logging::{LogStrategy, Logger}};
+use orchid_host::{extension::{init_systems, Extension}, lex::lex, tree::fmt_tt_v};
+
+#[derive(Parser, Debug)]
+#[command(version, about, long_about)]
+pub struct Args {
+  #[arg(short, long)]
+  extension: Vec<Utf8PathBuf>,
+  #[arg(short, long)]
+  system: Vec<String>,
+  #[command(subcommand)]
+  command: Commands,
+}
+
+#[derive(Subcommand, Debug)]
+pub enum Commands {
+  Lex{
+    #[arg(short, long)]
+    file: Utf8PathBuf
+  },
+}
+
+fn main() {
+  let args = Args::parse();
+  match args.command {
+    Commands::Lex { file } => {
+      let extensions = (args.extension.iter())
+        .map(|f| Extension::new(Command::new(f.as_os_str()), Logger::new(LogStrategy::StdErr)).unwrap())
+        .collect_vec();
+      let systems = init_systems(&args.system, &extensions).unwrap();
+      let mut file = File::open(file.as_std_path()).unwrap();
+      let mut buf = String::new();
+      file.read_to_string(&mut buf).unwrap();
+      let lexemes = lex(intern(&buf), &systems).unwrap();
+      println!("{}", fmt_tt_v(&lexemes))
+    }
+  }
 }