Lexer test mode works

orchid-base/src/char_filter.rs

@@ -1,11 +1,11 @@
-use std::ops::RangeInclusive;
+use std::{fmt, ops::RangeInclusive};
 
 use itertools::Itertools;
 use orchid_api::parser::CharFilter;
 
 pub type CRange = RangeInclusive<char>;
 
-pub trait ICFilter {
+pub trait ICFilter: fmt::Debug {
   fn ranges(&self) -> &[RangeInclusive<char>];
 }
 impl ICFilter for [RangeInclusive<char>] {
@@ -27,7 +27,7 @@ fn try_merge_char_ranges(left: CRange, right: CRange) -> Result<CRange, (CRange,
 pub fn mk_char_filter(items: impl IntoIterator<Item = CRange>) -> CharFilter {
   CharFilter(
     (items.into_iter())
-      .filter(|r| *r.start() as u32 + 1 < *r.end() as u32)
+      .filter(|r| *r.start() as u32 <= *r.end() as u32)
       .sorted_by_key(|r| *r.start() as u32)
       .coalesce(try_merge_char_ranges)
       .collect_vec(),

orchid-base/src/id_store.rs

@@ -24,6 +24,10 @@ impl<T> IdStore<T> {
     let id64 = id.into();
     if tbl_g.contains_key(&id64) { Some(IdRecord(id64, tbl_g)) } else { None }
   }
+  pub fn is_empty(&self) -> bool { self.len() == 0 }
+  pub fn len(&self) -> usize {
+    self.table.get().map(|t| t.lock().unwrap().len()).unwrap_or(0)
+  }
 }
 
 impl<T> Default for IdStore<T> {

orchid-base/src/interner.rs

@@ -315,11 +315,30 @@ pub fn sweep_master(retained: Retained) {
   g.interners.vecs.sweep_master(retained.vecs.into_iter().collect());
 }
 
-#[test]
-fn test_i() {
-  let _: Tok<String> = intern!(str: "foo");
-  let _: Tok<Vec<Tok<String>>> = intern!([Tok<String>]: &[
-    intern!(str: "bar"),
-    intern!(str: "baz")
-  ]);
+#[cfg(test)]
+mod test {
+  use super::*;
+
+    use std::num::NonZero;
+
+    use orchid_api::interner::TStr;
+    use orchid_api_traits::{Decode, Encode};
+
+  #[test]
+  fn test_i() {
+    let _: Tok<String> = intern!(str: "foo");
+    let _: Tok<Vec<Tok<String>>> = intern!([Tok<String>]: &[
+      intern!(str: "bar"),
+      intern!(str: "baz")
+    ]);
+  }
+
+  #[test]
+  fn test_coding() {
+    let coded = TStr(NonZero::new(3u64).unwrap());
+    let mut enc = &coded.enc_vec()[..];
+    eprintln!("Coded {enc:?}");
+    TStr::decode(&mut enc);
+    assert_eq!(enc, [])
+  }
 }

orchid-base/src/lib.rs

@@ -18,3 +18,4 @@ pub mod reqnot;
 pub mod sequence;
 pub mod tokens;
 pub mod tree;
+pub mod logging;

orchid-base/src/logging.rs

@@ -0,0 +1,16 @@
+use std::{fs::File, io::Write};
+
+pub use orchid_api::logging::LogStrategy;
+
+pub struct Logger(LogStrategy);
+impl Logger {
+  pub fn new(strat: LogStrategy) -> Self { Self(strat) }
+  pub fn log(&self, msg: String) {
+    match &self.0 {
+      LogStrategy::StdErr => eprintln!("{msg}"),
+      LogStrategy::File(f) => writeln!(File::open(f).unwrap(), "{msg}").unwrap(),
+    }
+  }
+  pub fn strat(&self) -> LogStrategy { self.0.clone() }
+}
+

orchid-base/src/msg.rs

@@ -1,15 +1,16 @@
+use orchid_api_traits::Decode;
 use std::io;
 
+use orchid_api_traits::Encode;
+
 pub fn send_msg(write: &mut impl io::Write, msg: &[u8]) -> io::Result<()> {
-  write.write_all(&(u32::try_from(msg.len()).unwrap()).to_be_bytes())?;
+  u32::try_from(msg.len()).unwrap().encode(write);
   write.write_all(msg)?;
   write.flush()
 }
 
 pub fn recv_msg(read: &mut impl io::Read) -> io::Result<Vec<u8>> {
-  let mut len = [0u8; 4];
-  read.read_exact(&mut len)?;
-  let len = u32::from_be_bytes(len);
+  let len = u32::decode(read);
   let mut msg = vec![0u8; len as usize];
   read.read_exact(&mut msg)?;
   Ok(msg)

orchid-base/src/reqnot.rs

@@ -1,5 +1,5 @@
 use std::marker::PhantomData;
-use std::mem;
+use std::{mem, thread};
 use std::ops::{BitAnd, Deref};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{sync_channel, SyncSender};
@@ -12,7 +12,7 @@ use trait_set::trait_set;
 
 trait_set! {
   pub trait SendFn<T: MsgSet> = for<'a> FnMut(&'a [u8], ReqNot<T>) + DynClone + Send + 'static;
-  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + DynClone + Send + 'static;
+  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + DynClone + Send + Sync + 'static;
   pub trait NotifFn<T: MsgSet> =
     for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + DynClone + Send + Sync + 'static;
 }
@@ -31,7 +31,7 @@ impl<MS: MsgSet + 'static> RequestHandle<MS> {
   pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
   pub fn req(&self) -> &<MS::In as Channel>::Req { &self.message }
   fn respond(&self, response: &impl Encode) {
-    assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded");
+    assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id);
     let mut buf = (!self.id).to_be_bytes().to_vec();
     response.encode(&mut buf);
     let mut send = clone_box(&*self.reqnot().0.lock().unwrap().send);
@@ -45,7 +45,8 @@ impl<MS: MsgSet + 'static> RequestHandle<MS> {
 }
 impl<MS: MsgSet> Drop for RequestHandle<MS> {
   fn drop(&mut self) {
-    debug_assert!(self.fulfilled.load(Ordering::Relaxed), "Request dropped without response")
+    let done = self.fulfilled.load(Ordering::Relaxed);
+    debug_assert!(done, "Request {} dropped without response", self.id)
   }
 }
 
@@ -63,6 +64,9 @@ pub struct ReqNotData<T: MsgSet> {
   responses: HashMap<u64, SyncSender<Vec<u8>>>,
 }
 
+/// Wraps a raw message buffer to save on copying.
+/// Dereferences to the tail of the message buffer, cutting off the ID
+#[derive(Debug, Clone)]
 pub struct RawReply(Vec<u8>);
 impl Deref for RawReply {
   type Target = [u8];
@@ -96,7 +100,8 @@ impl<T: MsgSet> ReqNot<T> {
       let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
       let mut req = clone_box(&*g.req);
       mem::drop(g);
-      req(RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() })
+      let handle = RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() };
+      thread::Builder::new().name(format!("request {id}")).spawn(move || req(handle)).unwrap();
     }
   }
 
@@ -109,6 +114,12 @@ impl<T: MsgSet> ReqNot<T> {
   }
 }
 
+pub trait DynRequester: Send + Sync {
+  type Transfer;
+  /// Encode and send a request, then receive the response buffer.
+  fn raw_request(&self, data: Self::Transfer) -> RawReply;
+}
+
 pub struct MappedRequester<'a, T>(Box<dyn Fn(T) -> RawReply + Send + Sync + 'a>);
 impl<'a, T> MappedRequester<'a, T> {
   fn new<U: DynRequester + 'a>(req: U) -> Self
@@ -139,10 +150,6 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
   }
 }
 
-pub trait DynRequester: Send + Sync {
-  type Transfer;
-  fn raw_request(&self, data: Self::Transfer) -> RawReply;
-}
 pub trait Requester: DynRequester {
   #[must_use = "These types are subject to change with protocol versions. \
     If you don't want to use the return value, At a minimum, force the type."]

orchid-base/src/tokens.rs

@@ -1,7 +1,12 @@
-use orchid_api::tree::{Placeholder, PlaceholderKind};
+use std::fmt::Display;
+
+use orchid_api::tree::{Paren, Placeholder, PlaceholderKind};
 
 use crate::interner::{deintern, Tok};
 
+pub const PARENS: &[(char, char, Paren)] =
+  &[('(', ')', Paren::Round), ('[', ']', Paren::Square), ('{', '}', Paren::Curly)];
+
 #[derive(Clone, Debug)]
 pub struct OwnedPh {
   pub name: Tok<String>,
@@ -13,3 +18,16 @@ impl OwnedPh {
   }
   pub fn from_api(ph: Placeholder) -> Self { Self { name: deintern(ph.name), kind: ph.kind } }
 }
+
+impl Display for OwnedPh {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    match self.kind {
+      PlaceholderKind::Name => write!(f, "$_{}", self.name),
+      PlaceholderKind::Scalar => write!(f, "${}", self.name),
+      PlaceholderKind::Vector { nz: false, prio: 0 } => write!(f, "..${}", self.name),
+      PlaceholderKind::Vector { nz: true, prio: 0 } => write!(f, "...${}", self.name),
+      PlaceholderKind::Vector { nz: false, prio } => write!(f, "..${}:{prio}", self.name),
+      PlaceholderKind::Vector { nz: true, prio } => write!(f, "...${}:{prio}", self.name),
+    }
+  }
+}