Changes in api and upwards

- Removed out-of-stack error reporting
- Revised module system to match previous Orchid system
- Errors are now in a Vec everywhere
- Implemented atoms and lexer
- Started implementation of line parser
- Tree is now ephemeral to avoid copying Atoms held inside
- Moved numbers into std and the shared parser into base
- Started implementation of Commands

orchid-std/src/lib.rs

@@ -1,3 +1,4 @@
+mod number;
 mod std;
 mod string;
 

orchid-std/src/number/mod.rs

@@ -0,0 +1,2 @@
+pub mod num_atom;
+pub mod num_lexer;

orchid-std/src/number/num_atom.rs

@@ -0,0 +1,61 @@
+use never::Never;
+use orchid_api_derive::Coding;
+use orchid_extension::atom::{Atomic, ReqPck, TypAtom};
+use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::error::{pack_err, ProjectResult};
+use orchid_extension::expr::{ExprHandle, GenExpr};
+use orchid_extension::system::SysCtx;
+use ordered_float::NotNan;
+
+#[derive(Clone, Debug, Coding)]
+pub struct Int(pub i64);
+impl Atomic for Int {
+  type Variant = ThinVariant;
+  type Data = Self;
+  type Req = Never;
+}
+impl ThinAtom for Int {
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+}
+impl TryFromExpr for Int {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    TypAtom::<Int>::try_from_expr(expr).map(|t| t.value)
+  }
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Float(pub NotNan<f64>);
+impl Atomic for Float {
+  type Variant = ThinVariant;
+  type Data = Self;
+  type Req = Never;
+}
+impl ThinAtom for Float {
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+}
+impl TryFromExpr for Float {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    TypAtom::<Float>::try_from_expr(expr).map(|t| t.value)
+  }
+}
+
+pub enum Numeric {
+  Int(i64),
+  Float(NotNan<f64>),
+}
+impl TryFromExpr for Numeric {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    Int::try_from_expr(expr.clone()).map(|t| Numeric::Int(t.0)).or_else(|e| {
+      Float::try_from_expr(expr).map(|t| Numeric::Float(t.0)).map_err(|e2| pack_err([e, e2]))
+    })
+  }
+}
+impl ToExpr for Numeric {
+  fn to_expr(self) -> GenExpr {
+    match self {
+      Self::Float(f) => Float(f).to_expr(),
+      Self::Int(i) => Int(i).to_expr(),
+    }
+  }
+}

orchid-std/src/number/num_lexer.rs

@@ -0,0 +1,44 @@
+use std::ops::RangeInclusive;
+
+use orchid_base::location::Pos;
+use orchid_base::number::{parse_num, NumError, NumErrorKind, Numeric};
+use orchid_extension::atom::AtomicFeatures;
+use orchid_extension::error::{ProjectError, ProjectResult};
+use orchid_extension::lexer::{LexContext, Lexer};
+use orchid_extension::tree::{GenTok, GenTokTree};
+use ordered_float::NotNan;
+
+use super::num_atom::{Float, Int};
+
+struct NumProjError(u32, NumError);
+impl ProjectError for NumProjError {
+  const DESCRIPTION: &'static str = "Failed to parse number";
+  fn message(&self) -> String {
+    match self.1.kind {
+      NumErrorKind::InvalidDigit => "This character is not meaningful in this base",
+      NumErrorKind::NaN => "Number somehow evaluated to NaN",
+      NumErrorKind::Overflow => "Number literal overflowed its enclosing type",
+    }
+    .to_string()
+  }
+  fn one_position(&self) -> Pos {
+    Pos::Range(self.0 + self.1.range.start as u32..self.0 + self.1.range.end as u32)
+  }
+}
+
+#[derive(Default)]
+pub struct NumLexer;
+impl Lexer for NumLexer {
+  const CHAR_FILTER: &'static [RangeInclusive<char>] = &['0'..='9'];
+  fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> ProjectResult<(&'a str, GenTokTree)> {
+    let ends_at = all.find(|c: char| !c.is_ascii_hexdigit() && !"xX._pP".contains(c));
+    let (chars, tail) = all.split_at(ends_at.unwrap_or(all.len()));
+    let fac = match parse_num(chars) {
+      Ok(Numeric::Float(f)) => Float(f).factory(),
+      Ok(Numeric::Uint(uint)) => Int(uint.try_into().unwrap()).factory(),
+      Ok(Numeric::Decimal(dec)) => Float(NotNan::new(dec.try_into().unwrap()).unwrap()).factory(),
+      Err(e) => return Err(NumProjError(ctx.pos(all), e).pack()),
+    };
+    Ok((tail, GenTok::Atom(fac).at(ctx.pos(all)..ctx.pos(tail))))
+  }
+}

orchid-std/src/std.rs

@@ -1,14 +1,16 @@
 use std::sync::Arc;
 
+use orchid_base::interner::Tok;
 use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
 use orchid_extension::fs::DeclFs;
 use orchid_extension::fun::Fun;
 use orchid_extension::system::{System, SystemCard};
 use orchid_extension::system_ctor::SystemCtor;
-use orchid_extension::tree::GenTree;
+use orchid_extension::tree::{cnst, module, root_mod, GenMemberKind};
 
-use crate::string::str_atom::StringAtom;
-use crate::string::str_leer::StringLexer;
+use crate::number::num_atom::{Float, Int};
+use crate::string::str_atom::StrAtom;
+use crate::string::str_lexer::StringLexer;
 use crate::OrcString;
 
 #[derive(Default)]
@@ -22,25 +24,23 @@ impl SystemCtor for StdSystem {
 }
 impl SystemCard for StdSystem {
   type Ctor = Self;
-  const ATOM_DEFS: &'static [Option<&'static dyn AtomDynfo>] = &[Some(StringAtom::INFO)];
+  const ATOM_DEFS: &'static [Option<&'static dyn AtomDynfo>] =
+    &[Some(Int::INFO), Some(Float::INFO), Some(StrAtom::INFO)];
 }
 impl System for StdSystem {
   fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }
   fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
-  fn env() -> GenTree {
-    GenTree::module([(
-      "std",
-      GenTree::module([(
-        "string",
-        GenTree::module([(
-          "concat",
-          GenTree::cnst(Fun::new(|left: OrcString| {
+  fn env() -> Vec<(Tok<String>, GenMemberKind)> {
+    vec![
+      root_mod("std", [], [
+        module(true, "string", [], [
+          cnst(true, "concat", Fun::new(|left: OrcString| {
             Fun::new(move |right: OrcString| {
-              StringAtom::new(Arc::new(left.get_string().to_string() + &right.get_string()))
+              StrAtom::new(Arc::new(left.get_string().to_string() + &right.get_string()))
             })
-          })),
-        )]),
-      )]),
-    )])
+          }))
+        ]),
+      ])
+    ]
   }
 }

orchid-std/src/string/mod.rs

@@ -1,2 +1,2 @@
 pub mod str_atom;
-pub mod str_leer;
+pub mod str_lexer;

orchid-std/src/string/str_atom.rs

@@ -2,93 +2,87 @@ use std::borrow::Cow;
 use std::num::NonZeroU64;
 use std::sync::Arc;
 
+use never::Never;
 use orchid_api::interner::TStr;
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Encode, Request};
 use orchid_base::id_store::IdStore;
 use orchid_base::interner::{deintern, Tok};
 use orchid_base::location::Pos;
-use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom::{Atomic, ReqPck, TypAtom};
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
 use orchid_extension::conv::TryFromExpr;
 use orchid_extension::error::{ProjectError, ProjectResult};
-use orchid_extension::expr::{ExprHandle, OwnedExpr};
-use orchid_extension::system::{downcast_atom, SysCtx};
+use orchid_extension::expr::ExprHandle;
+use orchid_extension::system::SysCtx;
 
 pub static STR_REPO: IdStore<Arc<String>> = IdStore::new();
 
-#[derive(Clone, Coding)]
-pub(crate) enum StringVal {
-  Val(NonZeroU64),
-  Int(TStr),
-}
 #[derive(Copy, Clone, Coding)]
-pub(crate) struct StringGetVal;
+pub struct StringGetVal;
 impl Request for StringGetVal {
   type Response = String;
 }
 
-pub(crate) enum StringAtom {
-  Val(NonZeroU64),
-  Int(Tok<String>),
-}
-impl Atomic for StringAtom {
+pub struct StrAtom(NonZeroU64);
+impl Atomic for StrAtom {
   type Variant = OwnedVariant;
-  type Data = StringVal;
+  type Data = NonZeroU64;
   type Req = StringGetVal;
 }
-impl StringAtom {
-  pub(crate) fn new_int(tok: Tok<String>) -> Self { Self::Int(tok) }
-  pub(crate) fn new(str: Arc<String>) -> Self { Self::Val(STR_REPO.add(str).id()) }
+impl StrAtom {
+  pub fn new(str: Arc<String>) -> Self { Self(STR_REPO.add(str).id()) }
 }
-impl Clone for StringAtom {
-  fn clone(&self) -> Self {
-    match &self {
-      Self::Int(t) => Self::Int(t.clone()),
-      Self::Val(v) => Self::Val(STR_REPO.add(STR_REPO.get(*v).unwrap().clone()).id()),
-    }
-  }
+impl Clone for StrAtom {
+  fn clone(&self) -> Self { Self(STR_REPO.add(STR_REPO.get(self.0).unwrap().clone()).id()) }
 }
-impl StringAtom {
-  fn try_local_value(&self) -> Option<Arc<String>> {
-    match self {
-      Self::Int(tok) => Some(tok.arc()),
-      Self::Val(id) => STR_REPO.get(*id).map(|r| r.clone()),
-    }
-  }
-  fn get_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
+impl StrAtom {
+  fn try_local_value(&self) -> Option<Arc<String>> { STR_REPO.get(self.0).map(|r| r.clone()) }
+  fn local_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
 }
-impl OwnedAtom for StringAtom {
-  fn val(&self) -> Cow<'_, Self::Data> {
-    Cow::Owned(match self {
-      Self::Int(tok) => StringVal::Int(tok.marker()),
-      Self::Val(id) => StringVal::Val(*id),
-    })
-  }
-  fn same(&self, _ctx: SysCtx, other: &Self) -> bool { self.get_value() == other.get_value() }
-  fn handle_req(
-    &self,
-    _ctx: SysCtx,
-    StringGetVal: Self::Req,
-    rep: &mut (impl std::io::Write + ?Sized),
-  ) {
-    self.get_value().encode(rep)
+impl OwnedAtom for StrAtom {
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0) }
+  fn same(&self, _ctx: SysCtx, other: &Self) -> bool { self.local_value() == other.local_value() }
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) {
+    self.local_value().encode(pck.unpack().1)
   }
 }
 
+#[derive(Debug, Clone)]
+pub struct IntStrAtom(Tok<String>);
+impl Atomic for IntStrAtom {
+  type Variant = OwnedVariant;
+  type Data = TStr;
+  type Req = Never;
+}
+impl From<Tok<String>> for IntStrAtom {
+  fn from(value: Tok<String>) -> Self { Self(value) }
+}
+impl OwnedAtom for IntStrAtom {
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.marker()) }
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+}
+
 #[derive(Clone)]
-pub struct OrcString(TypAtom<StringAtom>);
+pub enum OrcString {
+  Val(TypAtom<StrAtom>),
+  Int(Tok<String>),
+}
 impl OrcString {
   pub fn get_string(&self) -> Arc<String> {
-    match &self.0.value {
-      StringVal::Int(tok) => deintern(*tok).arc(),
-      StringVal::Val(id) => match STR_REPO.get(*id) {
+    match &self {
+      Self::Int(tok) => tok.arc(),
+      Self::Val(atom) => match STR_REPO.get(**atom) {
         Some(rec) => rec.clone(),
-        None => Arc::new(self.0.request(StringGetVal)),
+        None => Arc::new(atom.request(StringGetVal)),
       },
     }
   }
 }
+impl From<Tok<String>> for OrcString {
+  fn from(value: Tok<String>) -> Self { OrcString::Int(value) }
+}
+
 pub struct NotString(Pos);
 impl ProjectError for NotString {
   const DESCRIPTION: &'static str = "A string was expected";
@@ -96,9 +90,12 @@ impl ProjectError for NotString {
 }
 impl TryFromExpr for OrcString {
   fn try_from_expr(expr: ExprHandle) -> ProjectResult<OrcString> {
-    (OwnedExpr::new(expr).foreign_atom().map_err(|expr| expr.pos.clone()))
-      .and_then(|fatom| downcast_atom(fatom).map_err(|f| f.pos))
-      .map_err(|p| NotString(p).pack())
-      .map(OrcString)
+    if let Ok(v) = TypAtom::<StrAtom>::downcast(expr.clone()) {
+      return Ok(OrcString::Val(v));
+    }
+    match TypAtom::<IntStrAtom>::downcast(expr) {
+      Ok(t) => Ok(OrcString::Int(deintern(*t))),
+      Err(e) => Err(NotString(e.0).pack()),
+    }
   }
 }

orchid-std/src/string/str_lexer.rs

@@ -1,14 +1,14 @@
+use orchid_base::intern;
 use itertools::Itertools;
 use orchid_base::interner::intern;
 use orchid_base::location::Pos;
-use orchid_base::name::VName;
 use orchid_base::vname;
 use orchid_extension::atom::AtomicFeatures;
-use orchid_extension::error::{ErrorSansOrigin, ProjectErrorObj, ProjectResult};
-use orchid_extension::lexer::{LexContext, Lexer};
-use orchid_extension::tree::{wrap_tokv, OwnedTok, OwnedTokTree};
+use orchid_extension::error::{ErrorSansOrigin, ProjectError, ProjectErrorObj, ProjectResult};
+use orchid_extension::lexer::{LexContext, Lexer, NotApplicableLexerError};
+use orchid_extension::tree::{wrap_tokv, GenTok, GenTokTree};
 
-use super::str_atom::StringAtom;
+use super::str_atom::IntStrAtom;
 
 /// Reasons why [parse_string] might fail. See [StringError]
 #[derive(Clone)]
@@ -117,55 +117,48 @@ pub struct StringLexer;
 impl Lexer for StringLexer {
   const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"'];
   fn lex<'a>(
-    full_string: &'a str,
+    all: &'a str,
     ctx: &'a LexContext<'a>,
-  ) -> Option<ProjectResult<(&'a str, OwnedTokTree)>> {
-    full_string.strip_prefix('"').map(|mut tail| {
-      let mut parts = vec![];
-      let mut cur = String::new();
-      let commit_str = |str: &mut String, tail: &str, parts: &mut Vec<OwnedTokTree>| {
-        let str_val = parse_string(str)
-          .inspect_err(|e| ctx.report(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
-          .unwrap_or_default();
-        let tok = OwnedTok::Atom(StringAtom::new_int(intern(&str_val)).factory());
-        parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
-        *str = String::new();
-      };
-      loop {
-        if let Some(rest) = tail.strip_prefix('"') {
-          commit_str(&mut cur, tail, &mut parts);
-          return Ok((rest, wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
-        } else if let Some(rest) = tail.strip_prefix('$') {
-          commit_str(&mut cur, tail, &mut parts);
-          parts.push(OwnedTok::Name(VName::literal("++")).at(ctx.tok_ran(1, rest)));
-          parts.push(OwnedTok::Name(vname!(std::string::convert)).at(ctx.tok_ran(1, rest)));
-          match ctx.recurse(rest) {
-            Ok((new_tail, tree)) => {
-              tail = new_tail;
-              parts.push(tree);
-            },
-            Err(e) => {
-              ctx.report(e.clone());
-              return Ok(("", wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
-            },
-          }
-        } else if tail.starts_with('\\') {
-          // parse_string will deal with it, we just have to make sure we skip the next
-          // char
-          tail = &tail[2..];
+  ) -> ProjectResult<(&'a str, GenTokTree)> {
+    let mut tail = all.strip_prefix('"').ok_or_else(|| NotApplicableLexerError.pack())?;
+    let mut parts = vec![];
+    let mut cur = String::new();
+    let mut errors = vec![];
+    let commit_str = |str: &mut String, tail: &str, err: &mut Vec<ProjectErrorObj>, parts: &mut Vec<GenTokTree>| {
+      let str_val = parse_string(str)
+        .inspect_err(|e| err.push(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
+        .unwrap_or_default();
+      let tok = GenTok::Atom(IntStrAtom::from(intern(&*str_val)).factory());
+      parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
+      *str = String::new();
+    };
+    loop {
+      if let Some(rest) = tail.strip_prefix('"') {
+        commit_str(&mut cur, tail, &mut errors, &mut parts);
+        return Ok((rest, wrap_tokv(parts, ctx.pos(all)..ctx.pos(rest))));
+      } else if let Some(rest) = tail.strip_prefix('$') {
+        commit_str(&mut cur, tail, &mut errors, &mut parts);
+        parts.push(GenTok::Name(intern!(str: "++")).at(ctx.tok_ran(1, rest)));
+        parts.extend(GenTok::vname(&vname!(std::string::convert))
+          .map(|t| t.at(ctx.tok_ran(1, rest))));
+        let (new_tail, tree) = ctx.recurse(rest)?;
+        tail = new_tail;
+        parts.push(tree);
+      } else if tail.starts_with('\\') {
+        // parse_string will deal with it, we just have to make sure we skip the next
+        // char
+        tail = &tail[2..];
+      } else {
+        let mut ch = tail.chars();
+        if let Some(c) = ch.next() {
+          cur.push(c);
+          tail = ch.as_str();
         } else {
-          let mut ch = tail.chars();
-          if let Some(c) = ch.next() {
-            cur.push(c);
-            tail = ch.as_str();
-          } else {
-            let range = ctx.pos(full_string)..ctx.pos("");
-            commit_str(&mut cur, tail, &mut parts);
-            ctx.report(NoStringEnd.bundle(&Pos::Range(range.clone())));
-            return Ok(("", wrap_tokv(parts, range)));
-          }
+          let range = ctx.pos(all)..ctx.pos("");
+          commit_str(&mut cur, tail, &mut errors, &mut parts);
+          return Err(NoStringEnd.bundle(&Pos::Range(range.clone())));
         }
       }
-    })
+    }
   }
 }