Changes in api and upwards

- Removed out-of-stack error reporting
- Revised module system to match previous Orchid system
- Errors are now in a Vec everywhere
- Implemented atoms and lexer
- Started implementation of line parser
- Tree is now ephemeral to avoid copying Atoms held inside
- Moved numbers into std and the shared parser into base
- Started implementation of Commands

orchid-host/src/expr.rs

@@ -4,9 +4,9 @@ use std::sync::{Arc, RwLock};
 
 use hashbrown::HashMap;
 use lazy_static::lazy_static;
-use orchid_api::expr::ExprTicket;
+use orchid_api::expr::{Expr, ExprTicket};
 
-use crate::extension::AtomHand;
+use crate::extension::{AtomHand, System};
 
 #[derive(Clone, Debug)]
 pub struct RtExpr {
@@ -17,8 +17,10 @@ impl RtExpr {
   pub fn as_atom(&self) -> Option<AtomHand> { todo!() }
   pub fn strong_count(&self) -> usize { todo!() }
   pub fn id(&self) -> ExprTicket {
-    NonZeroU64::new(self.data.as_ref() as *const () as usize as u64)
-      .expect("this is a ref, it cannot be null")
+    ExprTicket(
+      NonZeroU64::new(self.data.as_ref() as *const () as usize as u64)
+        .expect("this is a ref, it cannot be null")
+    )
   }
   pub fn canonicalize(&self) -> ExprTicket {
     if !self.is_canonical.swap(true, Ordering::Relaxed) {
@@ -27,6 +29,7 @@ impl RtExpr {
     self.id()
   }
   pub fn resolve(tk: ExprTicket) -> Option<Self> { KNOWN_EXPRS.read().unwrap().get(&tk).cloned() }
+  pub fn from_api(api: Expr, sys: &System) -> Self { todo!() }
 }
 impl Drop for RtExpr {
   fn drop(&mut self) {

orchid-host/src/extension.rs

@@ -1,25 +1,27 @@
 use std::io::Write as _;
+use std::num::NonZero;
 use std::ops::Deref;
-use std::sync::atomic::{AtomicU16, AtomicU32, Ordering};
-use std::sync::mpsc::{channel, Receiver, Sender};
-use std::sync::{Arc, Mutex, RwLock, Weak};
+use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
+use std::sync::mpsc::{sync_channel, SyncSender};
+use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
 use std::{fmt, io, process, thread};
 
 use derive_destructure::destructure;
+use hashbrown::hash_map::Entry;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use orchid_api::atom::{Atom, AtomDrop, AtomSame, CallRef, FinalCall, Fwd, Fwded};
-use orchid_api::error::{ErrNotif, ProjErrOrRef, ProjResult, ReportError};
+use orchid_api::error::ProjResult;
 use orchid_api::expr::{Acquire, Expr, ExprNotif, ExprTicket, Release, Relocate};
 use orchid_api::interner::IntReq;
-use orchid_api::parser::{CharFilter, Lexed, SubLexed};
+use orchid_api::parser::{CharFilter, LexExpr, LexedExpr, ParsId, SubLex, SubLexed};
 use orchid_api::proto::{
   ExtHostNotif, ExtHostReq, ExtensionHeader, HostExtNotif, HostHeader, HostMsgSet,
 };
 use orchid_api::system::{NewSystem, SysDeclId, SysId, SystemDecl, SystemDrop};
-use orchid_api::tree::{GetConstTree, Tree, TreeId};
-use orchid_api_traits::{Coding, Decode, Encode};
+use orchid_api::tree::{GetMember, Member, MemberKind, TreeId};
+use orchid_api_traits::{Decode, Encode, Request};
 use orchid_base::char_filter::char_filter_match;
 use orchid_base::clone;
 use orchid_base::interner::{deintern, intern, Tok};
@@ -27,6 +29,7 @@ use orchid_base::reqnot::{ReqNot, Requester as _};
 use ordered_float::NotNan;
 
 use crate::expr::RtExpr;
+use crate::tree::OwnedMember;
 
 #[derive(Debug, destructure)]
 pub struct AtomData {
@@ -37,16 +40,16 @@ pub struct AtomData {
 impl AtomData {
   fn api(self) -> Atom {
     let (owner, drop, data) = self.destructure();
-    Atom { data, drop, owner: owner.0.id }
+    Atom { data, drop, owner: owner.id() }
   }
   fn api_ref(&self) -> Atom {
-    Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.0.id }
+    Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
   }
 }
 impl Drop for AtomData {
   fn drop(&mut self) {
-    self.owner.0.ext.0.reqnot.notify(AtomDrop(Atom {
-      owner: self.owner.0.id,
+    self.owner.reqnot().notify(AtomDrop(Atom {
+      owner: self.owner.id(),
       data: self.data.clone(),
       drop: true,
     }))
@@ -62,22 +65,22 @@ impl AtomHand {
   }
   pub fn call(self, arg: RtExpr) -> Expr {
     let owner_sys = self.0.owner.clone();
-    let ext = &owner_sys.0.ext;
+    let reqnot = owner_sys.reqnot();
     let ticket = owner_sys.give_expr(arg.canonicalize(), || arg);
     match Arc::try_unwrap(self.0) {
-      Ok(data) => ext.0.reqnot.request(FinalCall(data.api(), ticket)),
-      Err(hand) => ext.0.reqnot.request(CallRef(hand.api_ref(), ticket)),
+      Ok(data) => reqnot.request(FinalCall(data.api(), ticket)),
+      Err(hand) => reqnot.request(CallRef(hand.api_ref(), ticket)),
     }
   }
   pub fn same(&self, other: &AtomHand) -> bool {
-    let owner = self.0.owner.0.id;
-    if other.0.owner.0.id != owner {
+    let owner = self.0.owner.id();
+    if other.0.owner.id() != owner {
       return false;
     }
-    self.0.owner.0.ext.0.reqnot.request(AtomSame(self.0.api_ref(), other.0.api_ref()))
+    self.0.owner.reqnot().request(AtomSame(self.0.api_ref(), other.0.api_ref()))
   }
   pub fn req(&self, req: Vec<u8>) -> Vec<u8> {
-    self.0.owner.0.ext.0.reqnot.request(Fwded(self.0.api_ref(), req))
+    self.0.owner.reqnot().request(Fwded(self.0.api_ref(), req))
   }
   pub fn api_ref(&self) -> Atom { self.0.api_ref() }
 }
@@ -131,9 +134,7 @@ impl Extension {
             acq_expr(inc, expr);
             rel_expr(dec, expr);
           },
-          ExtHostNotif::ErrNotif(ErrNotif::ReportError(ReportError(sys, err))) => {
-            System::resolve(sys).unwrap().0.err_send.send(err).unwrap();
-          },
+          ExtHostNotif::AdviseSweep(_advice) => eprintln!("Sweep advice is unsupported")
         },
         |req| match req.req() {
           ExtHostReq::Ping(ping) => req.handle(ping, &()),
@@ -145,9 +146,16 @@ impl Extension {
           ExtHostReq::Fwd(fw @ Fwd(atom, _body)) => {
             let sys = System::resolve(atom.owner).unwrap();
             thread::spawn(clone!(fw; move || {
-              req.handle(&fw, &sys.0.ext.0.reqnot.request(Fwded(fw.0.clone(), fw.1.clone())))
+              req.handle(&fw, &sys.reqnot().request(Fwded(fw.0.clone(), fw.1.clone())))
             }));
           },
+          ExtHostReq::SubLex(sl) => {
+            let lex_g = LEX_RECUR.lock().unwrap();
+            let (rep_in, rep_out) = sync_channel(0);
+            let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
+            req_in.send(ReqPair(sl.clone(), rep_in)).unwrap();
+            req.handle(sl, &rep_out.recv().unwrap())
+          },
           _ => todo!(),
         },
       ),
@@ -169,23 +177,24 @@ impl SystemCtor {
   }
   pub fn run<'a>(&self, depends: impl IntoIterator<Item = &'a System>) -> System {
     let mut inst_g = SYSTEM_INSTS.write().unwrap();
-    let depends = depends.into_iter().map(|si| si.0.id).collect_vec();
+    let depends = depends.into_iter().map(|si| si.id()).collect_vec();
     debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
     let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
     static NEXT_ID: AtomicU16 = AtomicU16::new(1);
-    let id = SysId::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped");
+    let id = SysId(NonZero::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped"));
     let sys_inst = ext.reqnot.request(NewSystem { depends, id, system: self.decl.id });
-    let (err_send, err_rec) = channel();
     let data = System(Arc::new(SystemInstData {
       decl_id: self.decl.id,
       ext: Extension(ext),
       exprs: RwLock::default(),
       lex_filter: sys_inst.lex_filter,
-      const_root_id: sys_inst.const_root_id,
-      err_send,
-      err_rec: Mutex::new(err_rec),
+      const_root: OnceLock::new(),
       id,
     }));
+    let root = (sys_inst.const_root.into_iter())
+        .map(|(k, v)| OwnedMember::from_api(Member { public: true, name: k, kind: v }, &data))
+        .collect_vec();
+      data.0.const_root.set(root).unwrap();
     inst_g.insert(id, data.clone());
     data
   }
@@ -193,8 +202,11 @@ impl SystemCtor {
 
 lazy_static! {
   static ref SYSTEM_INSTS: RwLock<HashMap<SysId, System>> = RwLock::default();
+  static ref LEX_RECUR: Mutex<HashMap<ParsId, SyncSender<ReqPair<SubLex>>>> = Mutex::default();
 }
 
+pub struct ReqPair<R: Request>(R, pub SyncSender<R::Response>);
+
 #[derive(destructure)]
 pub struct SystemInstData {
   exprs: RwLock<HashMap<ExprTicket, (AtomicU32, RtExpr)>>,
@@ -202,9 +214,7 @@ pub struct SystemInstData {
   decl_id: SysDeclId,
   lex_filter: CharFilter,
   id: SysId,
-  const_root_id: TreeId,
-  err_rec: Mutex<Receiver<ProjErrOrRef>>,
-  err_send: Sender<ProjErrOrRef>,
+  const_root: OnceLock<Vec<OwnedMember>>,
 }
 impl Drop for SystemInstData {
   fn drop(&mut self) {
@@ -217,48 +227,59 @@ impl Drop for SystemInstData {
 #[derive(Clone)]
 pub struct System(Arc<SystemInstData>);
 impl System {
+  pub fn id(&self) -> SysId { self.id }
   fn resolve(id: SysId) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
+  fn reqnot(&self) -> &ReqNot<HostMsgSet> { &self.0.ext.0.reqnot }
   fn give_expr(&self, ticket: ExprTicket, get_expr: impl FnOnce() -> RtExpr) -> ExprTicket {
-    let mut exprs = self.0.exprs.write().unwrap();
-    exprs
-      .entry(ticket)
-      .and_modify(|(c, _)| {
-        c.fetch_add(1, Ordering::Relaxed);
-      })
-      .or_insert((AtomicU32::new(1), get_expr()));
+    match self.0.exprs.write().unwrap().entry(ticket) {
+      Entry::Occupied(mut oe) => {
+        oe.get_mut().0.fetch_add(1, Ordering::Relaxed);
+      },
+      Entry::Vacant(v) => {
+        v.insert((AtomicU32::new(1), get_expr()));
+      }
+    }
     ticket
   }
-  pub fn const_tree(&self) -> Tree {
-    self.0.ext.0.reqnot.request(GetConstTree(self.0.id, self.0.const_root_id))
-  }
-  pub fn catch_err<R: Coding>(&self, cb: impl FnOnce() -> ProjResult<R>) -> ProjResult<R> {
-    let mut errors = Vec::new();
-    if let Ok(err) = self.0.err_rec.lock().unwrap().try_recv() {
-      eprintln!("Errors left in queue");
-      errors.push(err);
-    }
-    let value = cb().inspect_err(|e| errors.extend(e.iter().cloned()));
-    while let Ok(err) = self.0.err_rec.lock().unwrap().try_recv() {
-      errors.push(err);
-    }
-    if !errors.is_empty() { Err(errors) } else { value }
+  pub fn get_tree(&self, id: TreeId) -> MemberKind {
+    self.reqnot().request(GetMember(self.0.id, id))
   }
   pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
   pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+  /// Have this system lex a part of the source. It is assumed that
+  /// [Self::can_lex] was called and returned true.
   pub fn lex(
     &self,
     source: Tok<String>,
-    pos: usize,
-    r: impl FnMut(usize) -> ProjResult<SubLexed>,
-  ) -> ProjResult<Lexed> {
-    todo!()
+    pos: u32,
+    mut r: impl FnMut(u32) -> Option<SubLexed> + Send,
+  ) -> ProjResult<Option<LexedExpr>> {
+    // get unique lex ID
+    static LEX_ID: AtomicU64 = AtomicU64::new(1);
+    let id = ParsId(NonZero::new(LEX_ID.fetch_add(1, Ordering::Relaxed)).unwrap());
+    thread::scope(|s| {
+      // create and register channel
+      let (req_in, req_out) = sync_channel(0);
+      LEX_RECUR.lock().unwrap().insert(id, req_in); // LEX_RECUR released
+      // spawn recursion handler which will exit when the sender is collected
+      s.spawn(move || {
+        while let Ok(ReqPair(sublex, rep_in)) = req_out.recv() {
+          rep_in.send(r(sublex.pos)).unwrap()
+        }
+      });
+      // Pass control to extension
+      let ret = self.reqnot().request(LexExpr { id, pos, sys: self.id(), text: source.marker() });
+      // collect sender to unblock recursion handler thread before returning
+      LEX_RECUR.lock().unwrap().remove(&id);
+      ret.transpose()
+    }) // exit recursion handler thread
   }
 }
 impl fmt::Debug for System {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     let ctor = (self.0.ext.0.systems.iter().find(|c| c.decl.id == self.0.decl_id))
       .expect("System instance with no associated constructor");
-    write!(f, "System({} @ {} #{}, ", ctor.decl.name, ctor.decl.priority, self.0.id)?;
+    write!(f, "System({} @ {} #{}, ", ctor.decl.name, ctor.decl.priority, self.0.id.0)?;
     match self.0.exprs.read() {
       Err(_) => write!(f, "expressions unavailable"),
       Ok(r) => {

orchid-host/src/lex.rs

@@ -1,44 +1,70 @@
-use orchid_api::tree::Paren;
-use orchid_base::intern;
-use orchid_base::interner::Tok;
-use orchid_base::location::Pos;
-use orchid_base::number::parse_num;
+use std::num::NonZeroU64;
 
-use crate::extension::System;
-use crate::results::{mk_err, num_to_err, OwnedResult};
+use hashbrown::HashMap;
+use orchid_api::parser::SubLexed;
+use orchid_api::system::SysId;
+use orchid_api::tree::{Paren, Token, TokenTree, TreeTicket};
+use orchid_base::error::OwnedError;
+use orchid_base::intern;
+use orchid_base::interner::{deintern, intern, Tok};
+use orchid_base::location::Pos;
+use orchid_base::tokens::OwnedPh;
+
+use crate::extension::{AtomHand, System};
+use crate::results::{mk_err, OwnedResult};
 use crate::tree::{OwnedTok, OwnedTokTree};
 
 pub struct LexCtx<'a> {
   pub systems: &'a [System],
-  pub source: Tok<String>,
-  pub src: &'a str,
+  pub source: &'a Tok<String>,
+  pub tail: &'a str,
+  pub sub_trees: &'a mut HashMap<TreeTicket, OwnedTokTree>,
 }
 impl<'a> LexCtx<'a> {
-  pub fn get_pos(&self) -> u32 { self.source.len() as u32 - self.src.len() as u32 }
+  pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
+  where 'a: 'b {
+    LexCtx {
+      source: self.source,
+      tail: &self.source[pos as usize..],
+      systems: self.systems,
+      sub_trees: &mut *self.sub_trees,
+    }
+  }
+  pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 }
+  pub fn end_pos(&self) -> u32 { self.source.len() as u32 }
+  pub fn set_pos(&mut self, pos: u32) { self.tail = &self.source[pos as usize..] }
+  pub fn push_pos(&mut self, delta: u32) { self.set_pos(self.get_pos() + delta) }
+  pub fn set_tail(&mut self, tail: &'a str) { self.tail = tail }
   pub fn strip_prefix(&mut self, tgt: &str) -> bool {
-    if let Some(src) = self.src.strip_prefix(tgt) {
-      self.src = src;
+    if let Some(src) = self.tail.strip_prefix(tgt) {
+      self.tail = src;
       return true;
     }
     false
   }
+  pub fn add_subtree(&mut self, subtree: OwnedTokTree) -> TreeTicket {
+    let next_idx = TreeTicket(NonZeroU64::new(self.sub_trees.len() as u64 + 1).unwrap());
+    self.sub_trees.insert(next_idx, subtree);
+    next_idx
+  }
+  pub fn rm_subtree(&mut self, ticket: TreeTicket) -> OwnedTokTree {
+    self.sub_trees.remove(&ticket).unwrap()
+  }
   pub fn strip_char(&mut self, tgt: char) -> bool {
-    if let Some(src) = self.src.strip_prefix(tgt) {
-      self.src = src;
+    if let Some(src) = self.tail.strip_prefix(tgt) {
+      self.tail = src;
       return true;
     }
     false
   }
   pub fn trim(&mut self, filter: impl Fn(char) -> bool) {
-    self.src = self.src.trim_start_matches(filter);
-  }
-  pub fn trim_ws(&mut self, br: bool) {
-    self.trim(|c| c.is_whitespace() && br || !"\r\n".contains(c))
+    self.tail = self.tail.trim_start_matches(filter);
   }
+  pub fn trim_ws(&mut self) { self.trim(|c| c.is_whitespace() && !"\r\n".contains(c)) }
   pub fn get_start_matches(&mut self, filter: impl Fn(char) -> bool) -> &'a str {
-    let rest = self.src.trim_start_matches(filter);
-    let matches = &self.src[..self.src.len() - rest.len()];
-    self.src = rest;
+    let rest = self.tail.trim_start_matches(filter);
+    let matches = &self.tail[..self.tail.len() - rest.len()];
+    self.tail = rest;
     matches
   }
 }
@@ -46,73 +72,129 @@ impl<'a> LexCtx<'a> {
 const PARENS: &[(char, char, Paren)] =
   &[('(', ')', Paren::Round), ('[', ']', Paren::Square), ('{', '}', Paren::Curly)];
 
-pub fn lex_tok(ctx: &mut LexCtx, br: bool) -> OwnedResult<OwnedTokTree> {
+pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
+  let start = ctx.get_pos();
   assert!(
-    !ctx.src.is_empty() && !ctx.src.starts_with(char::is_whitespace),
-    "Lexing empty string or whitespace to token! Invocations of lex_tok should check for empty string"
+    !ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space),
+    "Lexing empty string or whitespace to token!\n\
+    Invocations of lex_tok should check for empty string"
   );
-  for (open, close, paren) in PARENS {
-    let paren_pos = ctx.get_pos();
-    if ctx.strip_char(*open) {
-      let mut body = Vec::new();
-      return loop {
-        ctx.trim_ws(true);
-        if ctx.strip_char(*close) {
-          break Ok(OwnedTokTree {
-            tok: OwnedTok::S(paren.clone(), body),
-            range: paren_pos..ctx.get_pos(),
-          });
-        } else if ctx.src.is_empty() {
-          return Err(vec![mk_err(
-            intern!(str: "unclosed paren"),
-            format!("this {open} has no matching {close}"),
-            [Pos::Range(paren_pos..paren_pos + 1).into()],
-          )]);
-        }
-        body.push(lex_tok(ctx, true)?);
-      };
-    }
-  }
-  if ctx.strip_char('\\') {
-    let bs_pos = ctx.get_pos() - 1;
+  let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
+    OwnedTok::BR
+  } else if ctx.strip_prefix("::") {
+    OwnedTok::NS
+  } else if ctx.strip_prefix("--[") {
+    let (cmt, tail) = ctx.tail.split_once("]--").ok_or_else(|| {
+      vec![mk_err(
+        intern!(str: "Unterminated block comment"),
+        "This block comment has no ending ]--",
+        [Pos::Range(start..start + 3).into()],
+      )]
+    })?;
+    ctx.set_tail(tail);
+    OwnedTok::Comment(cmt.to_string())
+  } else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
+    let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
+    ctx.push_pos(end as u32);
+    OwnedTok::Comment(tail[2..end].to_string())
+  } else if ctx.strip_char('\\') {
     let mut arg = Vec::new();
-    loop {
-      ctx.trim_ws(true);
-      if ctx.strip_char('.') {
-        break;
-      } else if ctx.src.is_empty() {
+    ctx.trim_ws();
+    while !ctx.strip_char('.') {
+      if ctx.tail.is_empty() {
         return Err(vec![mk_err(
           intern!(str: "Unclosed lambda"),
           "Lambdae started with \\ should separate arguments from body with .",
-          [Pos::Range(bs_pos..bs_pos + 1).into()],
+          [Pos::Range(start..start + 1).into()],
         )]);
       }
-      arg.push(lex_tok(ctx, true)?);
+      arg.push(lex_once(ctx)?);
+      ctx.trim_ws();
     }
+    OwnedTok::Lambda(arg)
+  } else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
     let mut body = Vec::new();
-    return loop {
-      ctx.trim_ws(br);
-      let pos_before_end = ctx.get_pos();
-      if !br && ctx.strip_char('\n')
-        || PARENS.iter().any(|(_, e, _)| ctx.strip_char(*e))
-        || ctx.src.is_empty()
-      {
-        break Ok(OwnedTokTree { tok: OwnedTok::Lambda(arg, body), range: bs_pos..pos_before_end });
+    ctx.trim_ws();
+    while !ctx.strip_char(*rp) {
+      if ctx.tail.is_empty() {
+        return Err(vec![mk_err(
+          intern!(str: "unclosed paren"),
+          format!("this {lp} has no matching {rp}"),
+          [Pos::Range(start..start + 1).into()],
+        )]);
       }
-      body.push(lex_tok(ctx, br)?);
-    };
-  }
-  if ctx.src.starts_with(char::is_numeric) {
-    let num_pos = ctx.get_pos();
-    let num_str = ctx.get_start_matches(|c| c.is_alphanumeric() || "._".contains(c));
-    return Ok(OwnedTokTree {
-      range: num_pos..ctx.get_pos(),
-      tok: match parse_num(num_str) {
-        Err(e) => OwnedTok::Bottom(num_to_err(e, num_pos)),
-        Ok(v) => todo!(),
-      },
-    });
-  }
-  for sys in ctx.systems {}
-  todo!()
+      body.push(lex_once(ctx)?);
+      ctx.trim_ws();
+    }
+    OwnedTok::S(paren.clone(), body)
+  } else {
+    for sys in ctx.systems {
+      let mut errors = Vec::new();
+      if ctx.tail.starts_with(|c| sys.can_lex(c)) {
+        let lexed = sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| {
+          let mut sub_ctx = ctx.push(pos);
+          let ott = lex_once(&mut sub_ctx).inspect_err(|e| errors.extend(e.iter().cloned())).ok()?;
+          Some(SubLexed { pos: sub_ctx.get_pos(), ticket: sub_ctx.add_subtree(ott) })
+        });
+        match lexed {
+          Ok(None) if errors.is_empty() => continue,
+          Ok(None) => return Err(errors),
+          Err(e) => return Err(e.into_iter().map(|e| OwnedError::from_api(&e)).collect()),
+          Ok(Some(lexed)) => {
+            ctx.set_pos(lexed.pos);
+            return Ok(tt_to_owned(&lexed.expr, sys.id(), ctx))
+          },
+        }
+      }
+    }
+    if ctx.tail.starts_with(name_start) {
+      OwnedTok::Name(intern(ctx.get_start_matches(name_char)))
+    } else if ctx.tail.starts_with(op_char) {
+      OwnedTok::Name(intern(ctx.get_start_matches(op_char)))
+    } else {
+      return Err(vec![mk_err(
+        intern!(str: "Unrecognized character"),
+        "The following syntax is meaningless.",
+        [Pos::Range(start..start + 1).into()],
+      )]);
+    }
+  };
+  Ok(OwnedTokTree { tok, range: start..ctx.get_pos() })
 }
+
+fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
+fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
+fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}:\\".contains(c) }
+fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
+
+fn tt_to_owned(api: &TokenTree, sys: SysId, ctx: &mut LexCtx<'_>) -> OwnedTokTree {
+  let tok = match &api.token {
+    Token::Atom(atom) => OwnedTok::Atom(AtomHand::from_api(atom.clone().associate(sys))),
+    Token::Ph(ph) => OwnedTok::Ph(OwnedPh::from_api(ph.clone())),
+    Token::Bottom(err) => OwnedTok::Bottom(err.iter().map(OwnedError::from_api).collect()),
+    Token::Lambda(arg) => OwnedTok::Lambda(arg.iter().map(|t| tt_to_owned(t, sys, ctx)).collect()),
+    Token::Name(name) => OwnedTok::Name(deintern(*name)),
+    Token::S(p, b) => OwnedTok::S(p.clone(), b.iter().map(|t| tt_to_owned(t, sys, ctx)).collect()),
+    Token::Slot(id) => return ctx.rm_subtree(*id),
+    Token::BR => OwnedTok::BR,
+    Token::NS => OwnedTok::NS,
+  };
+  OwnedTokTree { range: api.range.clone(), tok }
+}
+
+pub fn lex(text: Tok<String>, systems: &[System]) -> OwnedResult<Vec<OwnedTokTree>> {
+  let mut sub_trees = HashMap::new();
+  let mut ctx = LexCtx {
+    source: &text,
+    sub_trees: &mut sub_trees,
+    tail: &text[..],
+    systems,
+  };
+  let mut tokv = Vec::new();
+  ctx.trim(unrep_space);
+  while !ctx.tail.is_empty() {
+    tokv.push(lex_once(&mut ctx)?);
+    ctx.trim(unrep_space);
+  }
+  Ok(tokv)
+}

orchid-host/src/lib.rs

@@ -4,3 +4,4 @@ pub mod extension;
 pub mod lex;
 pub mod results;
 pub mod tree;
+pub mod parse;

orchid-host/src/parse.rs

@@ -0,0 +1,33 @@
+use orchid_base::interner::Tok;
+
+use crate::tree::{OwnedItem, OwnedModule, OwnedTok, OwnedTokTree};
+
+pub struct ParseCtx<'a> {
+  tokens: &'a [OwnedTokTree]
+}
+
+pub fn split_br(ctx: ParseCtx) -> impl Iterator<Item = ParseCtx> {
+  ctx.tokens.split(|t| matches!(t.tok, OwnedTok::BR)).map(|tokens| ParseCtx { tokens })
+}
+
+pub fn strip_br(tt: &OwnedTokTree) -> Option<OwnedTokTree> {
+  let tok = match &tt.tok {
+    OwnedTok::BR => return None,
+    OwnedTok::Lambda(arg) => OwnedTok::Lambda(arg.iter().filter_map(strip_br).collect()),
+    OwnedTok::S(p, b) => OwnedTok::S(p.clone(), b.iter().filter_map(strip_br).collect()),
+    t => t.clone(),
+  };
+  Some(OwnedTokTree { tok, range: tt.range.clone() })
+}
+
+pub fn parse_items(ctx: ParseCtx) -> Vec<OwnedItem> {
+  todo!()
+}
+
+pub fn parse_item(ctx: ParseCtx) -> OwnedItem {
+  todo!()
+}
+
+pub fn parse_module(ctx: ParseCtx) -> (Tok<String>, OwnedModule) {
+  todo!()
+}

orchid-host/src/tree.rs

@@ -1,24 +1,159 @@
+use std::borrow::Borrow;
 use std::ops::Range;
+use std::sync::{Mutex, OnceLock};
 
-use orchid_api::tree::Paren;
+use itertools::Itertools;
+use never::Never;
+use orchid_api::tree::{Item, ItemKind, Macro, Member, MemberKind, Module, Paren, Token, TokenTree, TreeId, TreeTicket};
 use orchid_base::error::OwnedError;
-use orchid_base::name::VName;
+use orchid_base::interner::{deintern, Tok};
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
 use orchid_base::tokens::OwnedPh;
+use ordered_float::NotNan;
 
-use crate::extension::AtomHand;
+use crate::expr::RtExpr;
+use crate::extension::{AtomHand, System};
+use crate::results::OwnedResult;
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct OwnedTokTree {
   pub tok: OwnedTok,
   pub range: Range<u32>,
 }
+impl OwnedTokTree {
+  pub fn from_api<E>(
+    tt: &TokenTree,
+    sys: &System,
+    do_slot: &mut impl FnMut(&TreeTicket, Range<u32>) -> Result<Self, E>,
+  ) -> Result<Self, E> {
+    let tok = match &tt.token {
+      Token::Atom(a) => OwnedTok::Atom(AtomHand::from_api(a.clone().associate(sys.id()))),
+      Token::BR => OwnedTok::BR,
+      Token::NS => OwnedTok::NS,
+      Token::Bottom(e) => OwnedTok::Bottom(e.iter().map(OwnedError::from_api).collect_vec()),
+      Token::Lambda(arg) => OwnedTok::Lambda(Self::v_from_api(arg, sys, do_slot)?),
+      Token::Name(name) => OwnedTok::Name(deintern(*name)),
+      Token::Ph(ph) => OwnedTok::Ph(OwnedPh::from_api(ph.clone())),
+      Token::S(par, b) => OwnedTok::S(par.clone(), Self::v_from_api(b, sys, do_slot)?),
+      Token::Slot(id) => return do_slot(id, tt.range.clone()),
+    };
+    Ok(Self { range: tt.range.clone(), tok })
+  }
 
-#[derive(Clone)]
+  pub fn v_from_api<E>(
+    tokv: impl IntoIterator<Item: Borrow<TokenTree>>,
+    sys: &System,
+    do_slot: &mut impl FnMut(&TreeTicket, Range<u32>) -> Result<Self, E>,
+  ) -> Result<Vec<Self>, E> {
+    tokv.into_iter().map(|t| Self::from_api(t.borrow(), sys, do_slot)).collect()
+  }
+}
+
+#[derive(Clone, Debug)]
 pub enum OwnedTok {
-  Lambda(Vec<OwnedTokTree>, Vec<OwnedTokTree>),
-  Name(VName),
+  Comment(String),
+  Lambda(Vec<OwnedTokTree>),
+  Name(Tok<String>),
+  NS,
+  BR,
   S(Paren, Vec<OwnedTokTree>),
   Atom(AtomHand),
   Ph(OwnedPh),
-  Bottom(OwnedError),
+  Bottom(Vec<OwnedError>),
 }
+
+#[derive(Debug)]
+pub struct OwnedItem {
+  pub pos: Pos,
+  pub kind: OwnedItemKind,
+}
+
+#[derive(Debug)]
+pub enum OwnedItemKind {
+  Raw(Vec<OwnedTokTree>),
+  Member(OwnedMember),
+  Rule(OwnedMacro),
+}
+
+fn slot_panic(_: &TreeTicket, _: Range<u32>) -> Result<OwnedTokTree, Never> {
+  panic!("No slots allowed at item stage")
+}
+
+impl OwnedItem {
+  pub fn from_api(tree: Item, sys: &System) -> Self {
+    let kind = match tree.kind {
+      ItemKind::Raw(tokv) =>
+        OwnedItemKind::Raw(OwnedTokTree::v_from_api::<Never>(tokv, sys, &mut slot_panic).unwrap()),
+      ItemKind::Member(m) => OwnedItemKind::Member(OwnedMember::from_api(m, sys)),
+      ItemKind::Rule(r) => OwnedItemKind::Rule(OwnedMacro::from_api(r, sys)),
+    };
+    Self { pos: Pos::from_api(&tree.location), kind }
+  }
+}
+
+#[derive(Debug)]
+pub struct OwnedMember {
+  pub public: bool,
+  pub name: Tok<String>,
+  pub kind: OnceLock<OMemKind>,
+  pub lazy: Mutex<Option<LazyMemberHandle>>,
+}
+impl OwnedMember {
+  pub fn from_api(Member{ public, name, kind }: Member, sys: &System) -> Self {
+    let (kind, lazy) = match kind {
+      MemberKind::Const(c) => (OnceLock::from(OMemKind::Const(RtExpr::from_api(c, sys))), None),
+      MemberKind::Module(m) => (OnceLock::from(OMemKind::Mod(OwnedModule::from_api(m, sys))), None),
+      MemberKind::Lazy(id) => (OnceLock::new(), Some(LazyMemberHandle(id, sys.clone())))
+    };
+    OwnedMember { public, name: deintern(name), kind, lazy: Mutex::new(lazy) }
+  }
+}
+
+#[derive(Debug)]
+pub enum OMemKind {
+  Const(RtExpr),
+  Mod(OwnedModule),
+}
+
+#[derive(Debug)]
+pub struct OwnedModule {
+  pub imports: Vec<Sym>,
+  pub items: Vec<OwnedItem>,
+}
+impl OwnedModule {
+  pub fn from_api(m: Module, sys: &System) -> Self {
+    Self {
+      imports: m.imports.into_iter().map(|m| Sym::from_tok(deintern(m)).unwrap()).collect_vec(),
+      items: m.items.into_iter().map(|i| OwnedItem::from_api(i, sys)).collect_vec(),
+    }
+  }
+}
+
+#[derive(Debug)]
+pub struct OwnedMacro {
+  pub priority: NotNan<f64>,
+  pub pattern: Vec<OwnedTokTree>,
+  pub template: Vec<OwnedTokTree>,
+}
+impl OwnedMacro {
+  pub fn from_api(m: Macro, sys: &System) -> Self {
+    Self {
+      priority: m.priority,
+      pattern: OwnedTokTree::v_from_api(m.pattern, sys, &mut slot_panic).unwrap(),
+      template: OwnedTokTree::v_from_api(m.template, sys, &mut slot_panic).unwrap(),
+    }
+  }
+}
+
+#[derive(Debug)]
+pub struct LazyMemberHandle(TreeId, System);
+impl LazyMemberHandle {
+  pub fn run(self) -> OwnedResult<OMemKind> {
+    match self.1.get_tree(self.0) {
+      MemberKind::Const(c) => Ok(OMemKind::Const(RtExpr::from_api(c, &self.1))),
+      MemberKind::Module(m) => Ok(OMemKind::Mod(OwnedModule::from_api(m, &self.1))),
+      MemberKind::Lazy(id) => Self(id, self.1).run()
+    }
+  }
+}