very elegant extension API and parts of it used in std as POC

orchid-std/src/lib.rs

@@ -1,12 +1,6 @@
-pub fn add(left: usize, right: usize) -> usize { left + right }
+mod std;
+mod string;
 
-#[cfg(test)]
-mod tests {
-  use super::*;
+pub use std::StdSystem;
 
-  #[test]
-  fn it_works() {
-    let result = add(2, 2);
-    assert_eq!(result, 4);
-  }
-}
+pub use string::str_atom::OrcString;

orchid-std/src/main.rs

@@ -0,0 +1,4 @@
+use orchid_extension::entrypoint::{extension_main, ExtensionData};
+use orchid_std::StdSystem;
+
+pub fn main() { extension_main(ExtensionData { systems: &[&StdSystem] }) }

orchid-std/src/std.rs

@@ -0,0 +1,30 @@
+use orchid_extension::atom::owned_atom_info;
+use orchid_extension::fs::DeclFs;
+use orchid_extension::system::{System, SystemCard};
+use orchid_extension::system_ctor::SystemCtor;
+use orchid_extension::tree::GenTree;
+
+use crate::string::str_atom::StringAtom;
+use crate::string::str_leer::StringLexer;
+
+#[derive(Default)]
+pub struct StdSystem;
+impl SystemCtor for StdSystem {
+  type Deps = ();
+  type Instance = Self;
+  const NAME: &'static str = "orchid::std";
+  const VERSION: f64 = 0.00_01;
+  fn inst() -> Option<Self::Instance> { Some(StdSystem) }
+}
+impl SystemCard for StdSystem {
+  type Ctor = Self;
+  const ATOM_DEFS: &'static [Option<orchid_extension::atom::AtomInfo>] =
+    &[Some(owned_atom_info::<StringAtom>())];
+}
+impl System for StdSystem {
+  fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }
+  fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
+  fn env() -> GenTree {
+    GenTree::module([("std", GenTree::module([("string", GenTree::module([]))]))])
+  }
+}

orchid-std/src/string/mod.rs

@@ -0,0 +1,2 @@
+pub mod str_atom;
+pub mod str_leer;

orchid-std/src/string/str_atom.rs

@@ -0,0 +1,101 @@
+use std::borrow::Cow;
+use std::num::NonZeroU64;
+use std::sync::Arc;
+
+use orchid_api::interner::TStr;
+use orchid_api_derive::Coding;
+use orchid_api_traits::{Encode, Request};
+use orchid_base::id_store::IdStore;
+use orchid_base::interner::{deintern, Tok};
+use orchid_base::location::Pos;
+use orchid_extension::atom::{AtomCard, OwnedAtom, TypAtom};
+use orchid_extension::error::{ProjectError, ProjectResult};
+use orchid_extension::expr::{ExprHandle, OwnedExpr};
+use orchid_extension::system::{downcast_atom, SysCtx};
+use orchid_extension::try_from_expr::TryFromExpr;
+
+pub static STR_REPO: IdStore<Arc<String>> = IdStore::new();
+
+#[derive(Clone, Coding)]
+pub(crate) enum StringVal {
+  Val(NonZeroU64),
+  Int(TStr),
+}
+#[derive(Copy, Clone, Coding)]
+pub(crate) struct StringGetVal;
+impl Request for StringGetVal {
+  type Response = String;
+}
+
+pub(crate) enum StringAtom {
+  Val(NonZeroU64),
+  Int(Tok<String>),
+}
+impl AtomCard for StringAtom {
+  type Data = StringVal;
+  type Req = StringGetVal;
+}
+impl StringAtom {
+  pub(crate) fn new_int(tok: Tok<String>) -> Self { Self::Int(tok) }
+  pub(crate) fn new(str: Arc<String>) -> Self { Self::Val(STR_REPO.add(str).id()) }
+}
+impl Clone for StringAtom {
+  fn clone(&self) -> Self {
+    match &self {
+      Self::Int(t) => Self::Int(t.clone()),
+      Self::Val(v) => Self::Val(STR_REPO.add(STR_REPO.get(*v).unwrap().clone()).id()),
+    }
+  }
+}
+impl StringAtom {
+  fn try_local_value(&self) -> Option<Arc<String>> {
+    match self {
+      Self::Int(tok) => Some(tok.arc()),
+      Self::Val(id) => STR_REPO.get(*id).map(|r| r.clone()),
+    }
+  }
+  fn get_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
+}
+impl OwnedAtom for StringAtom {
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(match self {
+    Self::Int(tok) => StringVal::Int(tok.marker()),
+    Self::Val(id) => StringVal::Val(*id),
+  }) }
+  fn same(&self, _ctx: SysCtx, other: &Self) -> bool { self.get_value() == other.get_value() }
+  fn handle_req(
+    &self,
+    _ctx: SysCtx,
+    StringGetVal: Self::Req,
+    rep: &mut (impl std::io::Write + ?Sized),
+  ) {
+    self.get_value().encode(rep)
+  }
+}
+
+pub struct OrcString(TypAtom<StringAtom>);
+impl OrcString {
+  pub fn get_string(&self) -> Arc<String> {
+    match &self.0.value {
+      StringVal::Int(tok) => deintern(*tok).arc(),
+      StringVal::Val(id) => match STR_REPO.get(*id) {
+        Some(rec) => rec.clone(),
+        None => Arc::new(self.0.request(StringGetVal)),
+      },
+    }
+  }
+}
+pub struct NotString(Pos);
+impl ProjectError for NotString {
+  const DESCRIPTION: &'static str = "A string was expected";
+  fn one_position(&self) -> Pos {
+      self.0.clone()
+  }
+}
+impl TryFromExpr for OrcString {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    (OwnedExpr::new(expr).foreign_atom().map_err(|expr| expr.position.clone()))
+      .and_then(|fatom| downcast_atom(fatom).map_err(|f| f.position))
+      .map_err(|p| NotString(p).pack())
+      .map(OrcString)
+  }
+}

orchid-std/src/string/str_leer.rs

@@ -0,0 +1,171 @@
+use itertools::Itertools;
+use orchid_base::interner::intern;
+use orchid_base::location::Pos;
+use orchid_base::name::VName;
+use orchid_base::vname;
+use orchid_extension::atom::OwnedAtom;
+use orchid_extension::error::{ErrorSansOrigin, ProjectErrorObj, ProjectResult};
+use orchid_extension::lexer::{LexContext, Lexer};
+use orchid_extension::tree::{wrap_tokv, GenTok, GenTokTree};
+
+use super::str_atom::StringAtom;
+
+/// Reasons why [parse_string] might fail. See [StringError]
+#[derive(Clone)]
+enum StringErrorKind {
+  /// A unicode escape sequence wasn't followed by 4 hex digits
+  NotHex,
+  /// A unicode escape sequence contained an unassigned code point
+  BadCodePoint,
+  /// An unrecognized escape sequence was found
+  BadEscSeq,
+}
+
+/// Error produced by [parse_string]
+#[derive(Clone)]
+struct StringError {
+  /// Character where the error occured
+  pos: u32,
+  /// Reason for the error
+  kind: StringErrorKind,
+}
+
+#[derive(Clone)]
+struct NotHex;
+impl ErrorSansOrigin for NotHex {
+  const DESCRIPTION: &'static str = "Expected a hex digit";
+}
+
+#[derive(Clone)]
+struct BadCodePoint;
+impl ErrorSansOrigin for BadCodePoint {
+  const DESCRIPTION: &'static str = "The specified number is not a Unicode code point";
+}
+
+#[derive(Clone)]
+struct BadEscapeSequence;
+impl ErrorSansOrigin for BadEscapeSequence {
+  const DESCRIPTION: &'static str = "Unrecognized escape sequence";
+}
+
+impl StringError {
+  /// Convert into project error for reporting
+  pub fn into_proj(self, pos: u32) -> ProjectErrorObj {
+    let start = pos + self.pos;
+    let pos = Pos::Range(start..start + 1);
+    match self.kind {
+      StringErrorKind::NotHex => NotHex.bundle(&pos),
+      StringErrorKind::BadCodePoint => BadCodePoint.bundle(&pos),
+      StringErrorKind::BadEscSeq => BadEscapeSequence.bundle(&pos),
+    }
+  }
+}
+
+/// Process escape sequences in a string literal
+fn parse_string(str: &str) -> Result<String, StringError> {
+  let mut target = String::new();
+  let mut iter = str.char_indices().map(|(i, c)| (i as u32, c));
+  while let Some((_, c)) = iter.next() {
+    if c != '\\' {
+      target.push(c);
+      continue;
+    }
+    let (mut pos, code) = iter.next().expect("lexer would have continued");
+    let next = match code {
+      c @ ('\\' | '"' | '$') => c,
+      'b' => '\x08',
+      'f' => '\x0f',
+      'n' => '\n',
+      'r' => '\r',
+      't' => '\t',
+      '\n' => 'skipws: loop {
+        match iter.next() {
+          None => return Ok(target),
+          Some((_, c)) =>
+            if !c.is_whitespace() {
+              break 'skipws c;
+            },
+        }
+      },
+      'u' => {
+        let acc = ((0..4).rev())
+          .map(|radical| {
+            let (j, c) = (iter.next()).ok_or(StringError { pos, kind: StringErrorKind::NotHex })?;
+            pos = j;
+            let b = u32::from_str_radix(&String::from(c), 16)
+              .map_err(|_| StringError { pos, kind: StringErrorKind::NotHex })?;
+            Ok(16u32.pow(radical) + b)
+          })
+          .fold_ok(0, u32::wrapping_add)?;
+        char::from_u32(acc).ok_or(StringError { pos, kind: StringErrorKind::BadCodePoint })?
+      },
+      _ => return Err(StringError { pos, kind: StringErrorKind::BadEscSeq }),
+    };
+    target.push(next);
+  }
+  Ok(target)
+}
+
+#[derive(Clone)]
+pub struct NoStringEnd;
+impl ErrorSansOrigin for NoStringEnd {
+  const DESCRIPTION: &'static str = "String never terminated with \"";
+}
+
+#[derive(Default)]
+pub struct StringLexer;
+impl Lexer for StringLexer {
+  const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"'];
+  fn lex<'a>(
+    full_string: &'a str,
+    ctx: &'a LexContext<'a>,
+  ) -> Option<ProjectResult<(&'a str, GenTokTree)>> {
+    full_string.strip_prefix('"').map(|mut tail| {
+      let mut parts = vec![];
+      let mut cur = String::new();
+      let commit_str = |str: &mut String, tail: &str, parts: &mut Vec<GenTokTree>| {
+        let str_val = parse_string(str)
+          .inspect_err(|e| ctx.report(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
+          .unwrap_or_default();
+        let tok = GenTok::Atom(StringAtom::new_int(intern(&str_val)).factory());
+        parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
+        *str = String::new();
+      };
+      loop {
+        if let Some(rest) = tail.strip_prefix('"') {
+          commit_str(&mut cur, tail, &mut parts);
+          return Ok((rest, wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
+        } else if let Some(rest) = tail.strip_prefix('$') {
+          commit_str(&mut cur, tail, &mut parts);
+          parts.push(GenTok::Name(VName::literal("++")).at(ctx.tok_ran(1, rest)));
+          parts.push(GenTok::Name(vname!(std::string::convert)).at(ctx.tok_ran(1, rest)));
+          match ctx.recurse(rest) {
+            Ok((new_tail, tree)) => {
+              tail = new_tail;
+              parts.push(tree);
+            },
+            Err(e) => {
+              ctx.report(e.clone());
+              return Ok(("", wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
+            },
+          }
+        } else if tail.starts_with('\\') {
+          // parse_string will deal with it, we just have to make sure we skip the next
+          // char
+          tail = &tail[2..];
+        } else {
+          let mut ch = tail.chars();
+          if let Some(c) = ch.next() {
+            cur.push(c);
+            tail = ch.as_str();
+          } else {
+            let range = ctx.pos(full_string)..ctx.pos("");
+            commit_str(&mut cur, tail, &mut parts);
+            ctx.report(NoStringEnd.bundle(&Pos::Range(range.clone())));
+            return Ok(("", wrap_tokv(parts, range)));
+          }
+        }
+      }
+    })
+  }
+}