Method refactor now compiles

New constraint: custom lexer output is dropped whenever it is used to terminate an operator nested inside another custom lexer, because the recursive call has to return exactly one lexeme

.cargo/config.toml

@@ -1,7 +1,7 @@
 [alias]
 xtask = "run --quiet --package xtask --"
-orcx = "xtask orcx"
+orcx = "xtask orcx --"
-orcxdb = "xtask orcxdb"
+orcxdb = "xtask orcxdb --"
 
 [env]
 CARGO_WORKSPACE_DIR = { value = "", relative = true }

.gitea/workflows/rust.yml

@@ -2,17 +2,15 @@ name: Rust
 
 on:
   push:
-    branches: [ "master" ]
+    branches: [ "main" ]
   pull_request:
-    branches: [ "master" ]
+    branches: [ "main" ]
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
   build:
-    if: ${{ false }} # <- This make sure the workflow is skipped without any alert
-
     runs-on: ubuntu-latest
 
     steps:

Cargo.lock

@@ -948,10 +948,10 @@ dependencies = [
  "orchid-api",
  "orchid-api-traits",
  "orchid-base",
+ "orchid-extension",
  "ordered-float",
  "pastey",
  "substack",
- "test_executors",
  "tokio",
  "tokio-util",
  "trait-set",
@@ -981,6 +981,7 @@ dependencies = [
  "substack",
  "test_executors",
  "tokio",
+ "unicode-segmentation",
 ]
 
 [[package]]
@@ -1708,6 +1709,12 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+
 [[package]]
 name = "unicode-xid"
 version = "0.1.0"

README.md

@@ -22,7 +22,7 @@ Namespaces are inspired by Rust modules and ES6. Every file and directory is imp
 The project uses both the stable and nightly rust toolchain. Run the examples with
 
 ```sh
-cargo orcx -- exec --proj ./examples/hello-world "src::main::main"
+cargo orcx --release exec --proj ./examples/hello-world "src::main::main"
 ```
 
 you can try modifying the examples, but error reporting for the time being is pretty terrible.

examples/hello-world/main.orc

@@ -1,6 +1,4 @@
-let my_tuple = option::some t[1, 2]
+let user = r[ "foo" 1, "bar" t[3, 4] ]
+let _main = user.bar.1
 
-let main = match my_tuple {
+let main = "foo" + string::slice "hello" 1 3 + "bar"
-	option::some t[ref head, ..] => head;
-	option::none => "foo";
-}

orchid-api/src/lexer.rs

@@ -28,7 +28,7 @@ impl Request for LexExpr {
 #[derive(Clone, Debug, Coding)]
 pub struct LexedExpr {
 	pub pos: u32,
-	pub expr: TokenTree,
+	pub expr: Vec<TokenTree>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-base/Cargo.toml

@@ -12,7 +12,10 @@ async-once-cell = "0.5.4"
 bound = "0.6.0"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.20"
-futures = { version = "0.3.31", features = ["std"], default-features = false }
+futures = { version = "0.3.31", default-features = false, features = [
+	"std",
+	"async-await",
+] }
 hashbrown = "0.16.1"
 itertools = "0.14.0"
 lazy_static = "1.5.0"

orchid-base/src/future_debug.rs

@@ -5,6 +5,7 @@ use std::rc::Rc;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::task::{Context, Poll, Wake, Waker};
+use std::thread::panicking;
 
 use futures::Stream;
 use itertools::Itertools;
@@ -155,3 +156,15 @@ pub fn spin_on<Fut: Future>(f: Fut) -> Fut::Output {
 		}
 	}
 }
+
+/// Create an object that will panic if dropped. [PanicOnDrop::defuse] must be
+/// called once the particular constraint preventing a drop has passed
+pub fn assert_no_drop(msg: &'static str) -> PanicOnDrop { PanicOnDrop(true, msg) }
+
+pub struct PanicOnDrop(bool, &'static str);
+impl PanicOnDrop {
+	pub fn defuse(mut self) { self.0 = false; }
+}
+impl Drop for PanicOnDrop {
+	fn drop(&mut self) { assert!(panicking() || !self.0, "{}", self.1) }
+}

orchid-base/src/lib.rs

@@ -28,6 +28,6 @@ pub mod reqnot;
 pub mod sequence;
 pub mod side;
 pub mod stash;
-mod tl_cache;
+pub mod tl_cache;
 pub mod tokens;
 pub mod tree;

orchid-base/src/name.rs

@@ -198,7 +198,8 @@ impl Sym {
 		let items = v.into_iter().collect_vec();
 		Self::from_tok(iv(&items).await)
 	}
-	/// Read a `::` separated namespaced name.
+	/// Read a `::` separated namespaced name. Do not use this for statically
+	/// known names, use the [sym] macro instead which is cached.
 	pub async fn parse(s: &str) -> Result<Self, EmptyNameError> {
 		Ok(Sym(iv(&VName::parse(s).await?.into_vec()).await))
 	}
@@ -285,20 +286,28 @@ impl NameLike for VName {}
 
 /// Create a [Sym] literal.
 ///
-/// Both the name and its components will be cached in a thread-local static so
+/// The name and its components will be cached in a thread-local static so
 /// that subsequent executions of the expression only incur an Arc-clone for
 /// cloning the token.
 #[macro_export]
 macro_rules! sym {
   ($seg1:tt $( :: $seg:tt)*) => {
-		$crate::name::Sym::from_tok(
+		$crate::tl_cache!(async $crate::name::Sym : {
-			$crate::interner::iv(&[
+			$crate::name::Sym::from_tok(
-				$crate::interner::is(stringify!($seg1)).await
+				$crate::interner::iv(&[
-				$( , $crate::interner::is(stringify!($seg)).await )*
+					$crate::interner::is($crate::sym!(@SEG $seg1)).await
-			])
+					$( , $crate::interner::is($crate::sym!(@SEG $seg)).await )*
-			.await
+				])
-		).unwrap()
+				.await
+			).unwrap()
+		})
   };
+	(@SEG [ $($data:tt)* ]) => {
+		stringify!($($data)*)
+	};
+	(@SEG $data:tt) => {
+		stringify!($data)
+	};
 }
 
 /// Create a [VName] literal.
@@ -307,10 +316,12 @@ macro_rules! sym {
 #[macro_export]
 macro_rules! vname {
   ($seg1:tt $( :: $seg:tt)*) => {
-    $crate::name::VName::new([
+		$crate::tl_cache!(async $crate::name::VName : {
-      $crate::interner::is(stringify!($seg1)).await
+			$crate::name::VName::new([
-      $( , $crate::interner::is(stringify!($seg)).await )*
+				$crate::interner::is(stringify!($seg1)).await
-    ]).unwrap()
+				$( , $crate::interner::is(stringify!($seg)).await )*
+			]).unwrap()
+		})
 	};
 }
 
@@ -320,10 +331,12 @@ macro_rules! vname {
 #[macro_export]
 macro_rules! vpath {
   ($seg1:tt $( :: $seg:tt)*) => {
-    $crate::name::VPath(vec![
+		$crate::tl_cache!(async $crate::name::VPath : {
-      $crate::interner::is(stringify!($seg1)).await
+			$crate::name::VPath(vec![
-      $( , $crate::interner::is(stringify!($seg)).await )*
+				$crate::interner::is(stringify!($seg1)).await
-		])
+				$( , $crate::interner::is(stringify!($seg)).await )*
+			])
+		})
 	};
   () => {
     $crate::name::VPath(vec![])
@@ -334,30 +347,43 @@ macro_rules! vpath {
 pub mod test {
 	use std::borrow::Borrow;
 
+	use orchid_api_traits::spin_on;
+
 	use super::{NameLike, Sym, VName};
-	use crate::interner::{IStr, is};
+	use crate::interner::local_interner::local_interner;
+	use crate::interner::{IStr, is, with_interner};
 	use crate::name::VPath;
 
-	pub async fn recur() {
+	#[test]
-		let myname = vname!(foo::bar);
+	pub fn recur() {
-		let _borrowed_slice: &[IStr] = myname.borrow();
+		spin_on(with_interner(local_interner(), async {
-		let _deref_pathslice: &[IStr] = &myname;
+			let myname = vname!(foo::bar);
-		let _as_slice_out: &[IStr] = myname.as_slice();
+			let _borrowed_slice: &[IStr] = myname.borrow();
+			let _deref_pathslice: &[IStr] = &myname;
+			let _as_slice_out: &[IStr] = myname.as_slice();
+		}))
 	}
 
 	/// Tests that literals are correctly interned as equal
-	pub async fn literals() {
+	#[test]
-		assert_eq!(
+	pub fn literals() {
-			sym!(foo::bar::baz),
+		spin_on(with_interner(local_interner(), async {
-			Sym::new([is("foo").await, is("bar").await, is("baz").await]).await.unwrap()
+			assert_eq!(
-		);
+				sym!(foo::bar::baz),
-		assert_eq!(
+				Sym::new([is("foo").await, is("bar").await, is("baz").await]).await.unwrap()
-			vname!(foo::bar::baz),
+			);
-			VName::new([is("foo").await, is("bar").await, is("baz").await]).unwrap()
+			assert_eq!(
-		);
+				sym!(foo::bar::[|>]),
-		assert_eq!(
+				Sym::new([is("foo").await, is("bar").await, is("|>").await]).await.unwrap()
-			vpath!(foo::bar::baz),
+			);
-			VPath::new([is("foo").await, is("bar").await, is("baz").await])
+			assert_eq!(
-		);
+				vname!(foo::bar::baz),
+				VName::new([is("foo").await, is("bar").await, is("baz").await]).unwrap()
+			);
+			assert_eq!(
+				{ vpath!(foo::bar::baz) },
+				VPath::new([is("foo").await, is("bar").await, is("baz").await])
+			);
+		}))
 	}
 }

orchid-base/src/parse.rs

@@ -16,7 +16,7 @@ use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_fmt, ttv_range}
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
 pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
-pub fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}\\".contains(c) }
+pub fn op_char(c: char) -> bool { !name_char(c) && !unrep_space(c) && !"()[]{}\\".contains(c) }
 pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 
 /// A cheaply copiable subsection of a document that holds onto context data and

orchid-base/src/reqnot.rs

@@ -17,6 +17,7 @@ use futures::{
 use hashbrown::HashMap;
 use orchid_api_traits::{Decode, Encode, Request, UnderRoot};
 
+use crate::future_debug::{PanicOnDrop, assert_no_drop};
 use crate::localset::LocalSet;
 
 #[must_use = "Receipts indicate that a required action has been performed within a function. \
@@ -238,9 +239,12 @@ impl IoClient {
 impl Client for IoClient {
 	fn start_notif(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn MsgWriter<'_> + '_>>> {
 		Box::pin(async {
+			let drop_g = assert_no_drop("Notif future dropped");
 			let mut o = self.lock_out().await;
 			0u64.encode(o.as_mut()).await?;
-			Ok(Box::new(IoNotifWriter { o }) as Box<dyn MsgWriter>)
+			drop_g.defuse();
+			Ok(Box::new(IoNotifWriter { o, drop_g: assert_no_drop("Notif writer dropped") })
+				as Box<dyn MsgWriter>)
 		})
 	}
 	fn start_request(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn ReqWriter<'_> + '_>>> {
@@ -252,11 +256,17 @@ impl Client for IoClient {
 			};
 			let (cb, reply) = oneshot::channel();
 			let (ack, got_ack) = oneshot::channel();
+			let drop_g = assert_no_drop("Request future dropped");
 			self.subscribe.as_ref().clone().send(ReplySub { id, ack, cb }).await.unwrap();
 			got_ack.await.unwrap();
 			let mut w = self.lock_out().await;
 			id.encode(w.as_mut()).await?;
-			Ok(Box::new(IoReqWriter { reply, w }) as Box<dyn ReqWriter>)
+			drop_g.defuse();
+			Ok(Box::new(IoReqWriter {
+				reply,
+				w,
+				drop_g: assert_no_drop("Request reader dropped without reply"),
+			}) as Box<dyn ReqWriter>)
 		})
 	}
 }
@@ -264,36 +274,49 @@ impl Client for IoClient {
 struct IoReqWriter {
 	reply: oneshot::Receiver<IoGuard<dyn AsyncRead>>,
 	w: IoGuard<dyn AsyncWrite>,
+	drop_g: PanicOnDrop,
 }
 impl<'a> ReqWriter<'a> for IoReqWriter {
 	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.w.as_mut() }
 	fn send(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepReader<'a> + 'a>>> {
 		Box::pin(async {
-			let Self { reply, mut w } = *self;
+			let Self { reply, mut w, drop_g } = *self;
 			w.flush().await?;
 			mem::drop(w);
 			let i = reply.await.expect("Client dropped before reply received");
-			Ok(Box::new(IoRepReader { i }) as Box<dyn RepReader>)
+			drop_g.defuse();
+			Ok(Box::new(IoRepReader {
+				i,
+				drop_g: assert_no_drop("Reply reader dropped without finishing"),
+			}) as Box<dyn RepReader>)
 		})
 	}
 }
 
 struct IoRepReader {
 	i: IoGuard<dyn AsyncRead>,
+	drop_g: PanicOnDrop,
 }
 impl<'a> RepReader<'a> for IoRepReader {
 	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { self.i.as_mut() }
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'static, ()> { Box::pin(async {}) }
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'static, ()> {
+		Box::pin(async { self.drop_g.defuse() })
+	}
 }
 
 #[derive(destructure)]
 struct IoNotifWriter {
 	o: IoGuard<dyn AsyncWrite>,
+	drop_g: PanicOnDrop,
 }
 impl<'a> MsgWriter<'a> for IoNotifWriter {
 	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.o.as_mut() }
 	fn finish(mut self: Box<Self>) -> LocalBoxFuture<'static, io::Result<()>> {
-		Box::pin(async move { self.o.flush().await })
+		Box::pin(async move {
+			self.o.flush().await?;
+			self.drop_g.defuse();
+			Ok(())
+		})
 	}
 }
 
@@ -375,7 +398,8 @@ impl IoCommServer {
 					Ok(Event::Exit) => break,
 					Ok(Event::Sub(ReplySub { id, ack, cb })) => {
 						pending_replies.insert(id, cb);
-						ack.send(()).unwrap();
+						// this is detected and logged on client
+						let _ = ack.send(());
 					},
 					Ok(Event::Input(0, read)) => {
 						let notif = &notif;

orchid-base/src/tl_cache.rs

@@ -6,4 +6,18 @@ macro_rules! tl_cache {
 		}
 		V.with(|v| v.clone())
 	}};
+	(async $ty:ty : $expr:expr) => {{
+		type CellType = std::cell::OnceCell<$ty>;
+		thread_local! {
+			static V: CellType = std::cell::OnceCell::default();
+		}
+		match V.with(|cell: &CellType| cell.get().cloned()) {
+			Some(val) => val as $ty,
+			None => {
+				let val = $expr;
+				let _ = V.with(|cell: &CellType| cell.set(val.clone()));
+				val as $ty
+			},
+		}
+	}};
 }

orchid-extension/src/atom.rs

@@ -1,7 +1,9 @@
 use std::any::{Any, TypeId, type_name};
 use std::collections::HashMap;
-use std::fmt;
+use std::fmt::{self, Debug};
 use std::future::Future;
+use std::io;
+use std::marker::PhantomData;
 use std::num::NonZeroU32;
 use std::ops::Deref;
 use std::pin::Pin;
@@ -9,14 +11,15 @@ use std::rc::Rc;
 
 use dyn_clone::{DynClone, clone_box};
 use futures::future::LocalBoxFuture;
-use futures::{AsyncRead, AsyncWrite, FutureExt, StreamExt, stream};
+use futures::{AsyncWrite, FutureExt, StreamExt, stream};
 use orchid_api_derive::Coding;
-use orchid_api_traits::{Coding, Decode, Encode, Request, enc_vec};
+use orchid_api_traits::{Coding, Decode, InHierarchy, Request, UnderRoot, enc_vec};
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv, mk_errv_floating};
-use orchid_base::format::{FmtCtx, FmtUnit, Format, fmt};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, fmt, take_first};
 use orchid_base::interner::is;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
+use orchid_base::reqnot::{Receipt, ReqHandle, ReqReader, ReqReaderExt};
 use trait_set::trait_set;
 
 use crate::api;
@@ -98,14 +101,17 @@ impl ForeignAtom {
 	pub(crate) fn new(handle: Rc<ExprHandle>, atom: api::Atom, pos: Pos) -> Self {
 		ForeignAtom { atom, expr: handle, pos }
 	}
-	pub async fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
+	pub async fn request<R: Request + UnderRoot<Root: AtomMethod>>(
+		&self,
+		r: R,
+	) -> Option<R::Response> {
 		let rep = (request(api::Fwd(
 			self.atom.clone(),
-			Sym::parse(M::NAME).await.unwrap().tok().to_api(),
+			Sym::parse(<R as UnderRoot>::Root::NAME).await.unwrap().tok().to_api(),
-			enc_vec(&m),
+			enc_vec(&r.into_root()),
 		)))
 		.await?;
-		Some(M::Response::decode_slice(&mut &rep[..]))
+		Some(R::Response::decode_slice(&mut &rep[..]))
 	}
 	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TAtom<T>, NotTypAtom> {
 		TAtom::downcast(self.ex().handle()).await
@@ -144,24 +150,50 @@ impl NotTypAtom {
 		)
 	}
 }
+impl Debug for NotTypAtom {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		f.debug_struct("NotTypAtom")
+			.field("pos", &self.pos)
+			.field("expr", &self.expr)
+			.field("typ.name", &self.typ.name())
+			.finish_non_exhaustive()
+	}
+}
 
-pub trait AtomMethod: Request + Coding {
+pub trait AtomMethod: Coding + InHierarchy {
 	const NAME: &str;
 }
 pub trait Supports<M: AtomMethod>: AtomCard {
-	fn handle(&self, req: M) -> impl Future<Output = <M as Request>::Response>;
+	fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: M,
+	) -> impl Future<Output = io::Result<Receipt<'a>>>;
 }
 
-trait_set! {
+trait HandleAtomMethod<A> {
-	trait AtomReqCb<A> = for<'a> Fn(
+	fn handle<'a, 'b: 'a>(
-		&'a A,
+		&'a self,
-		Pin<&'a mut dyn AsyncRead>,
+		atom: &'a A,
-		Pin<&'a mut dyn AsyncWrite>,
+		req: Box<dyn ReqReader<'b> + 'a>,
-	) -> LocalBoxFuture<'a, ()>
+	) -> LocalBoxFuture<'a, ()>;
+}
+struct AtomMethodHandler<M, A>(PhantomData<M>, PhantomData<A>);
+impl<M: AtomMethod, A: Supports<M>> HandleAtomMethod<A> for AtomMethodHandler<M, A> {
+	fn handle<'a, 'b: 'a>(
+		&'a self,
+		a: &'a A,
+		mut reader: Box<dyn ReqReader<'b> + 'a>,
+	) -> LocalBoxFuture<'a, ()> {
+		Box::pin(async {
+			let req = reader.read_req::<M>().await.unwrap();
+			let _ = Supports::<M>::handle(a, reader.finish().await, req).await.unwrap();
+		})
+	}
 }
 
 pub struct MethodSetBuilder<A: AtomCard> {
-	handlers: Vec<(&'static str, Rc<dyn AtomReqCb<A>>)>,
+	handlers: Vec<(&'static str, Rc<dyn HandleAtomMethod<A>>)>,
 }
 impl<A: AtomCard> MethodSetBuilder<A> {
 	pub fn new() -> Self { Self { handlers: vec![] } }
@@ -169,15 +201,7 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 	pub fn handle<M: AtomMethod>(mut self) -> Self
 	where A: Supports<M> {
 		assert!(!M::NAME.is_empty(), "AtomMethod::NAME cannoot be empty");
-		self.handlers.push((
+		self.handlers.push((M::NAME, Rc::new(AtomMethodHandler::<M, A>(PhantomData, PhantomData))));
-			M::NAME,
-			Rc::new(move |a: &A, req: Pin<&mut dyn AsyncRead>, rep: Pin<&mut dyn AsyncWrite>| {
-				async {
-					Supports::<M>::handle(a, M::decode(req).await.unwrap()).await.encode(rep).await.unwrap()
-				}
-				.boxed_local()
-			}),
-		));
 		self
 	}
 
@@ -192,20 +216,19 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 }
 
 pub struct MethodSet<A: AtomCard> {
-	handlers: HashMap<Sym, Rc<dyn AtomReqCb<A>>>,
+	handlers: HashMap<Sym, Rc<dyn HandleAtomMethod<A>>>,
 }
 impl<A: AtomCard> MethodSet<A> {
 	pub(crate) async fn dispatch<'a>(
-		&'a self,
+		&self,
-		atom: &'a A,
+		atom: &'_ A,
 		key: Sym,
-		req: Pin<&'a mut dyn AsyncRead>,
+		req: Box<dyn ReqReader<'a> + 'a>,
-		rep: Pin<&'a mut dyn AsyncWrite>,
 	) -> bool {
 		match self.handlers.get(&key) {
 			None => false,
 			Some(handler) => {
-				handler(atom, req, rep).await;
+				handler.handle(atom, req).await;
 				true
 			},
 		}
@@ -234,13 +257,13 @@ impl<A: AtomicFeatures> TAtom<A> {
 			},
 		}
 	}
-	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
+	pub async fn request<R: Request + UnderRoot<Root: AtomMethod>>(&self, req: R) -> R::Response
-	where A: Supports<M> {
+	where A: Supports<<R as UnderRoot>::Root> {
-		M::Response::decode_slice(
+		R::Response::decode_slice(
 			&mut &(request(api::Fwd(
 				self.untyped.atom.clone(),
-				Sym::parse(M::NAME).await.unwrap().tok().to_api(),
+				Sym::parse(<R as UnderRoot>::Root::NAME).await.unwrap().tok().to_api(),
-				enc_vec(&req),
+				enc_vec(&req.into_root()),
 			)))
 			.await
 			.unwrap()[..],
@@ -269,12 +292,11 @@ pub trait AtomDynfo: 'static {
 	fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
 	fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
 	fn print<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit>;
-	fn handle_req<'a, 'b: 'a, 'c: 'a>(
+	fn handle_req<'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'b mut dyn AsyncRead>,
+		req: Box<dyn ReqReader<'a> + 'a>,
-		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool>;
 	fn command<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>>;
 	fn serialize<'a, 'b: 'a>(
@@ -311,12 +333,18 @@ impl Format for AtomFactory {
 	}
 }
 
-pub async fn err_not_callable() -> OrcErrv {
+pub async fn err_not_callable(unit: &FmtUnit) -> OrcErrv {
-	mk_errv_floating(is("This atom is not callable").await, "Attempted to apply value as function")
+	mk_errv_floating(
+		is("This atom is not callable").await,
+		format!("Attempted to apply {} as function", take_first(unit, false)),
+	)
 }
 
-pub async fn err_not_command() -> OrcErrv {
+pub async fn err_not_command(unit: &FmtUnit) -> OrcErrv {
-	mk_errv_floating(is("This atom is not a command").await, "Settled on an inactionable value")
+	mk_errv_floating(
+		is("This atom is not a command").await,
+		format!("Settled on {} which is an inactionable value", take_first(unit, false)),
+	)
 }
 
 /// Read the type ID prefix from an atom, return type information and the rest

orchid-extension/src/atom_owned.rs

@@ -11,7 +11,7 @@ use std::rc::Rc;
 use async_once_cell::OnceCell;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::{LocalBoxFuture, ready};
-use futures::{AsyncRead, AsyncWrite, FutureExt};
+use futures::{AsyncWrite, FutureExt};
 use futures_locks::{RwLock, RwLockReadGuard};
 use itertools::Itertools;
 use memo_map::MemoMap;
@@ -119,17 +119,16 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn print(&self, AtomCtx(_, id): AtomCtx<'_>) -> LocalBoxFuture<'_, FmtUnit> {
 		Box::pin(async move { AtomReadGuard::new(id.unwrap()).await.dyn_print().await })
 	}
-	fn handle_req<'a, 'b: 'a, 'c: 'a>(
+	fn handle_req<'a>(
 		&'a self,
-		AtomCtx(_, id): AtomCtx,
+		AtomCtx(_, id): AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'b mut dyn AsyncRead>,
+		req: Box<dyn orchid_base::reqnot::ReqReader<'a> + 'a>,
-		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let a = AtomReadGuard::new(id.unwrap()).await;
 			let ms = self.ms.get_or_init(self.msbuild.pack()).await;
-			ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), key, req, rep).await
+			ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), key, req).await
 		})
 	}
 	fn command<'a>(
@@ -228,7 +227,7 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	fn val(&self) -> impl Future<Output = Cow<'_, Self::Data>>;
 	#[allow(unused_variables)]
 	fn call_ref(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot(err_not_callable().await) }
+		async move { bot(err_not_callable(&self.dyn_print().await).await) }
 	}
 	fn call(self, arg: Expr) -> impl Future<Output = GExpr> {
 		async {
@@ -239,7 +238,7 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	}
 	#[allow(unused_variables)]
 	fn command(self) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command().await) }
+		async move { Err(err_not_command(&self.dyn_print().await).await) }
 	}
 	#[allow(unused_variables)]
 	fn free(self) -> impl Future<Output = ()> { async {} }

orchid-extension/src/atom_thin.rs

@@ -4,7 +4,7 @@ use std::pin::Pin;
 
 use async_once_cell::OnceCell;
 use futures::future::LocalBoxFuture;
-use futures::{AsyncRead, AsyncWrite, FutureExt};
+use futures::{AsyncWrite, FutureExt};
 use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
@@ -54,16 +54,15 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn call_ref<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
 		Box::pin(async move { T::decode_slice(&mut &buf[..]).call(arg).await })
 	}
-	fn handle_req<'a, 'm1: 'a, 'm2: 'a>(
+	fn handle_req<'a>(
 		&'a self,
-		AtomCtx(buf, _): AtomCtx<'a>,
+		AtomCtx(buf, ..): AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'m1 mut dyn AsyncRead>,
+		req: Box<dyn orchid_base::reqnot::ReqReader<'a> + 'a>,
-		rep: Pin<&'m2 mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack()).await;
-			ms.dispatch(&T::decode_slice(&mut &buf[..]), key, req, rep).await
+			ms.dispatch(&T::decode_slice(&mut &buf[..]), key, req).await
 		})
 	}
 	fn command<'a>(
@@ -99,11 +98,11 @@ pub trait ThinAtom:
 {
 	#[allow(unused_variables)]
 	fn call(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot(err_not_callable().await) }
+		async move { bot(err_not_callable(&self.print().await).await) }
 	}
 	#[allow(unused_variables)]
 	fn command(&self) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command().await) }
+		async move { Err(err_not_command(&self.print().await).await) }
 	}
 	#[allow(unused_variables)]
 	fn print(&self) -> impl Future<Output = FmtUnit> {

orchid-extension/src/conv.rs

@@ -4,13 +4,14 @@ use std::pin::Pin;
 use dyn_clone::DynClone;
 use never::Never;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::format::{Format, fmt};
 use orchid_base::interner::is;
 use orchid_base::location::Pos;
 use trait_set::trait_set;
 
-use crate::atom::{AtomicFeatures, ForeignAtom, TAtom, ToAtom};
+use crate::atom::{AtomicFeatures, ForeignAtom, TAtom};
-use crate::expr::Expr;
+use crate::expr::{Expr, ExprKind};
-use crate::gen_expr::{GExpr, atom, bot};
+use crate::gen_expr::{GExpr, bot};
 
 pub trait TryFromExpr: Sized {
 	fn try_from_expr(expr: Expr) -> impl Future<Output = OrcRes<Self>>;
@@ -26,14 +27,17 @@ impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
 	}
 }
 
-async fn err_not_atom(pos: Pos) -> OrcErrv {
+async fn err_not_atom(pos: Pos, value: &impl Format) -> OrcErrv {
-	mk_errv(is("Expected an atom").await, "This expression is not an atom", [pos])
+	mk_errv(is("Expected an atom").await, format!("{} is not an atom", fmt(value).await), [pos])
 }
 
 impl TryFromExpr for ForeignAtom {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
-		match expr.atom().await {
+		if let ExprKind::Bottom(err) = &expr.data().await.kind {
-			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone()).await),
+			return Err(err.clone());
+		}
+		match expr.clone().atom().await {
+			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &expr).await),
 			Ok(f) => Ok(f),
 		}
 	}
@@ -107,10 +111,6 @@ impl<T: ToExpr> ToExpr for OrcRes<T> {
 	}
 }
 
-impl<A: ToAtom> ToExpr for A {
-	async fn to_gen(self) -> GExpr { atom(self) }
-}
-
 impl ToExpr for Never {
 	async fn to_gen(self) -> GExpr { match self {} }
 }

orchid-extension/src/coroutine_exec.rs

@@ -13,7 +13,7 @@ use crate::atom::Atomic;
 use crate::atom_owned::{OwnedAtom, OwnedVariant};
 use crate::conv::{ToExpr, TryFromExpr};
 use crate::expr::Expr;
-use crate::gen_expr::{GExpr, arg, call, lambda, seq};
+use crate::gen_expr::{GExpr, arg, call, lambda, new_atom, seq};
 
 enum Command {
 	Execute(GExpr, Sender<Expr>),
@@ -34,11 +34,11 @@ impl BuilderCoroutine {
 			None => panic!("Before the stream ends, we should have gotten a Halt"),
 			Some(Command::Halt(expr)) => expr,
 			Some(Command::Execute(expr, reply)) => call(
-				lambda(0, [seq([arg(0)], call(Replier { reply, builder: self }.to_gen().await, [arg(0)]))]),
+				lambda(0, [seq([arg(0)], call(new_atom(Replier { reply, builder: self }), [arg(0)]))]),
 				[expr],
 			),
 			Some(Command::Register(expr, reply)) =>
-				call(Replier { reply, builder: self }.to_gen().await, [expr]),
+				call(new_atom(Replier { reply, builder: self }), [expr]),
 		}
 	}
 }

orchid-extension/src/entrypoint.rs

@@ -1,12 +1,12 @@
 use std::cell::RefCell;
 use std::future::Future;
+use std::mem;
 use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
-use std::{io, mem};
 
 use futures::future::{LocalBoxFuture, join_all};
-use futures::{AsyncRead, AsyncWrite, AsyncWriteExt, StreamExt, stream};
+use futures::{AsyncWriteExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_api_traits::{Decode, Encode, Request, UnderRoot, enc_vec};
@@ -17,8 +17,7 @@ use orchid_base::logging::{log, with_logger};
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Snippet};
 use orchid_base::reqnot::{
-	Client, ClientExt, CommCtx, MsgReader, MsgReaderExt, Receipt, RepWriter, ReqHandle, ReqHandleExt,
+	Client, ClientExt, CommCtx, MsgReader, MsgReaderExt, ReqHandleExt, ReqReaderExt, Witness, io_comm,
-	ReqReader, ReqReaderExt, Witness, io_comm,
 };
 use orchid_base::stash::with_stash;
 use orchid_base::tree::{TokenVariant, ttv_from_api};
@@ -39,6 +38,7 @@ use crate::reflection::with_refl_roots;
 use crate::system::{SysCtx, atom_by_idx, cted, with_sys};
 use crate::system_ctor::{CtedObj, DynSystemCtor, SystemCtor};
 use crate::tree::{TreeIntoApiCtxImpl, get_lazy, with_lazy_member_store};
+use crate::trivial_req::TrivialReqCycle;
 
 task_local::task_local! {
 	static CLIENT: Rc<dyn Client>;
@@ -219,36 +219,6 @@ impl ExtensionBuilder {
 								let fwd_tok = Witness::of(&fwd);
 								let api::SysFwded(sys_id, payload) = fwd;
 								with_sys_record(sys_id, async {
-									struct TrivialReqCycle<'a> {
-										req: &'a [u8],
-										rep: &'a mut Vec<u8>,
-									}
-									impl<'a> ReqReader<'a> for TrivialReqCycle<'a> {
-										fn reader(&mut self) -> Pin<&mut dyn AsyncRead> {
-											Pin::new(&mut self.req) as Pin<&mut _>
-										}
-										fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>> {
-											Box::pin(async { self as Box<_> })
-										}
-									}
-									impl<'a> ReqHandle<'a> for TrivialReqCycle<'a> {
-										fn start_reply(
-											self: Box<Self>,
-										) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>> {
-											Box::pin(async { Ok(self as Box<_>) })
-										}
-									}
-									impl<'a> RepWriter<'a> for TrivialReqCycle<'a> {
-										fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> {
-											Pin::new(&mut self.rep) as Pin<&mut _>
-										}
-										fn finish(
-											self: Box<Self>,
-										) -> LocalBoxFuture<'a, io::Result<orchid_base::reqnot::Receipt<'a>>>
-										{
-											Box::pin(async { Ok(Receipt::_new()) })
-										}
-									}
 									let mut reply = Vec::new();
 									let req = TrivialReqCycle { req: &payload, rep: &mut reply };
 									let _ = cted().inst().dyn_request(Box::new(req)).await;
@@ -261,7 +231,8 @@ impl ExtensionBuilder {
 									let text = es(text).await;
 									let src = Sym::from_api(src).await;
 									let expr_store = BorrowedExprStore::new();
-									let trigger_char = text.chars().nth(pos as usize).unwrap();
+									let tail = &text[pos as usize..];
+									let trigger_char = tail.chars().next().unwrap();
 									let ekey_na = ekey_not_applicable().await;
 									let ekey_cascade = ekey_cascade().await;
 									let lexers = cted().inst().dyn_lexers();
@@ -269,7 +240,7 @@ impl ExtensionBuilder {
 										lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char))
 									{
 										let ctx = LexContext::new(&expr_store, &text, id, pos, src.clone());
-										match try_with_reporter(lx.lex(&text[pos as usize..], &ctx)).await {
+										match try_with_reporter(lx.lex(tail, &ctx)).await {
 											Err(e) if e.any(|e| *e == ekey_na) => continue,
 											Err(e) => {
 												let eopt = e.keep_only(|e| *e != ekey_cascade).map(|e| Err(e.to_api()));
@@ -277,7 +248,11 @@ impl ExtensionBuilder {
 												return handle.reply(&lex, &eopt).await;
 											},
 											Ok((s, expr)) => {
-												let expr = expr.into_api(&mut (), &mut ()).await;
+												let expr = join_all(
+													(expr.into_iter())
+														.map(|tok| async { tok.into_api(&mut (), &mut ()).await }),
+												)
+												.await;
 												let pos = (text.len() - s.len()) as u32;
 												expr_store.dispose().await;
 												return handle.reply(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
@@ -344,14 +319,8 @@ impl ExtensionBuilder {
 											let api::Fwded(_, key, payload) = &fwded;
 											let mut reply = Vec::new();
 											let key = Sym::from_api(*key).await;
-											let some = nfo
+											let req = TrivialReqCycle { req: payload, rep: &mut reply };
-												.handle_req(
+											let some = nfo.handle_req(actx, key, Box::new(req)).await;
-													actx,
-													key,
-													Pin::<&mut &[u8]>::new(&mut &payload[..]),
-													Pin::<&mut Vec<_>>::new(&mut reply),
-												)
-												.await;
 											handle.reply(fwded, &some.then_some(reply)).await
 										},
 										api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {

orchid-extension/src/expr.rs

@@ -136,6 +136,7 @@ impl Expr {
 			_ => Err(self),
 		}
 	}
+	pub async fn pos(&self) -> Pos { self.data().await.pos.clone() }
 	pub fn handle(&self) -> Rc<ExprHandle> { self.handle.clone() }
 
 	pub fn slot(&self) -> GExpr {

orchid-extension/src/func_atom.rs

@@ -6,7 +6,7 @@ use std::future::Future;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use futures::future::LocalBoxFuture;
+use futures::future::{LocalBoxFuture, join_all};
 use futures::{AsyncWrite, FutureExt};
 use itertools::Itertools;
 use never::Never;
@@ -14,6 +14,7 @@ use orchid_api_traits::Encode;
 use orchid_base::clone;
 use orchid_base::error::OrcRes;
 use orchid_base::format::{FmtCtx, FmtUnit};
+use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use task_local::task_local;
 use trait_set::trait_set;
@@ -24,13 +25,36 @@ use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
 use crate::coroutine_exec::{ExecHandle, exec};
 use crate::expr::Expr;
-use crate::gen_expr::GExpr;
+use crate::gen_expr::{GExpr, new_atom};
 use crate::system::sys_id;
 
 trait_set! {
 	trait FunCB = Fn(Vec<Expr>) -> LocalBoxFuture<'static, OrcRes<GExpr>> + 'static;
 }
 
+task_local! {
+	static ARGV: Vec<Expr>;
+}
+
+pub fn get_arg(idx: usize) -> Expr {
+	ARGV
+		.try_with(|argv| {
+			(argv.get(idx).cloned())
+				.unwrap_or_else(|| panic!("Cannot read argument ##{idx}, only have {}", argv.len()))
+		})
+		.expect("get_arg called outside ExprFunc")
+}
+
+pub fn get_argc() -> usize {
+	ARGV.try_with(|argv| argv.len()).expect("get_arg called outside ExprFunc")
+}
+
+pub async fn get_arg_posv(idxes: impl IntoIterator<Item = usize>) -> impl Iterator<Item = Pos> {
+	let args = (ARGV.try_with(|argv| idxes.into_iter().map(|i| &argv[i]).cloned().collect_vec()))
+		.expect("get_arg_posv called outside ExprFunc");
+	join_all(args.iter().map(|expr| expr.pos())).await.into_iter()
+}
+
 pub trait ExprFunc<I, O>: Clone + 'static {
 	fn argtyps() -> &'static [TypeId];
 	fn apply<'a>(&self, hand: ExecHandle<'a>, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
@@ -110,7 +134,7 @@ impl OwnedAtom for Fun {
 		if new_args.len() == self.record.argtyps.len() {
 			(self.record.fun)(new_args).await.to_gen().await
 		} else {
-			Self { args: new_args, record: self.record.clone(), path: self.path.clone() }.to_gen().await
+			new_atom(Self { args: new_args, record: self.record.clone(), path: self.path.clone() })
 		}
 	}
 	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
@@ -156,7 +180,7 @@ impl OwnedAtom for Lambda {
 		if new_args.len() == self.record.argtyps.len() {
 			(self.record.fun)(new_args).await.to_gen().await
 		} else {
-			Self { args: new_args, record: self.record.clone() }.to_gen().await
+			new_atom(Self { args: new_args, record: self.record.clone() })
 		}
 	}
 	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
@@ -168,7 +192,7 @@ mod expr_func_derives {
 
 	use orchid_base::error::OrcRes;
 
-	use super::ExprFunc;
+	use super::{ARGV, ExprFunc};
 	use crate::conv::{ToExpr, TryFromExpr};
 	use crate::func_atom::{ExecHandle, Expr};
 	use crate::gen_expr::GExpr;
@@ -187,8 +211,9 @@ mod expr_func_derives {
           }
           async fn apply<'a>(&self, _: ExecHandle<'a>, v: Vec<Expr>) -> OrcRes<GExpr> {
             assert_eq!(v.len(), Self::argtyps().len(), "Arity mismatch");
+						let argv = v.clone();
             let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above"));
-            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_gen().await)
+            Ok(ARGV.scope(argv, self($($t::try_from_expr([< $t:lower >]).await?,)*)).await.to_gen().await)
           }
         }
       }

orchid-extension/src/gen_expr.rs

@@ -106,7 +106,8 @@ impl Format for GExprKind {
 fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
 
 pub fn sym_ref(path: Sym) -> GExpr { inherit(GExprKind::Const(path)) }
-pub fn atom<A: ToAtom>(atom: A) -> GExpr { inherit(GExprKind::NewAtom(atom.to_atom_factory())) }
+/// Creates an expression from a new atom that we own.
+pub fn new_atom<A: ToAtom>(atom: A) -> GExpr { inherit(GExprKind::NewAtom(atom.to_atom_factory())) }
 
 pub fn seq(deps: impl IntoIterator<Item = GExpr>, val: GExpr) -> GExpr {
 	fn recur(mut ops: impl Iterator<Item = GExpr>) -> Option<GExpr> {

orchid-extension/src/lexer.rs

@@ -72,12 +72,25 @@ impl<'a> LexContext<'a> {
 	}
 }
 
+pub trait LexedData {
+	fn into_vec(self) -> Vec<GenTokTree>;
+}
+impl LexedData for GenTokTree {
+	fn into_vec(self) -> Vec<GenTokTree> { vec![self] }
+}
+impl LexedData for Vec<GenTokTree> {
+	fn into_vec(self) -> Vec<GenTokTree> { self }
+}
+impl<const N: usize> LexedData for [GenTokTree; N] {
+	fn into_vec(self) -> Vec<GenTokTree> { self.to_vec() }
+}
+
 pub trait Lexer: Debug + Send + Sync + Sized + Default + 'static {
 	const CHAR_FILTER: &'static [RangeInclusive<char>];
 	fn lex<'a>(
 		tail: &'a str,
 		lctx: &'a LexContext<'a>,
-	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
+	) -> impl Future<Output = OrcRes<(&'a str, impl LexedData)>>;
 }
 
 pub trait DynLexer: Debug + Send + Sync + 'static {
@@ -86,7 +99,7 @@ pub trait DynLexer: Debug + Send + Sync + 'static {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>>;
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -95,8 +108,8 @@ impl<T: Lexer> DynLexer for T {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>> {
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>> {
-		T::lex(tail, ctx).boxed_local()
+		async { T::lex(tail, ctx).await.map(|(s, d)| (s, d.into_vec())) }.boxed_local()
 	}
 }
 

orchid-extension/src/lib.rs

@@ -17,7 +17,9 @@ pub mod logger;
 pub mod other_system;
 pub mod parser;
 pub mod reflection;
+pub mod stream_reqs;
 pub mod system;
 pub mod system_ctor;
 pub mod tokio;
 pub mod tree;
+mod trivial_req;

orchid-extension/src/parser.rs

@@ -184,12 +184,12 @@ impl ConstCtx {
 		&'b self,
 		names: impl IntoIterator<Item = &'b Sym> + 'b,
 	) -> impl Stream<Item = OrcRes<Sym>> + 'b {
-		let resolve_names = api::ResolveNames {
+		let names = names.into_iter().map(|n| n.to_api()).collect_vec();
-			constid: self.constid,
-			sys: sys_id(),
-			names: names.into_iter().map(|n| n.to_api()).collect_vec(),
-		};
 		stream(async |mut cx| {
+			if names.is_empty() {
+				return;
+			}
+			let resolve_names = api::ResolveNames { constid: self.constid, sys: sys_id(), names };
 			for name_opt in request(resolve_names).await {
 				cx.emit(match name_opt {
 					Err(e) => Err(OrcErrv::from_api(&e).await),

orchid-extension/src/reflection.rs

@@ -47,8 +47,9 @@ impl ReflMod {
 	async fn try_populate(&self) -> Result<(), api::LsModuleError> {
 		let path_tok = iv(&self.0.path[..]).await;
 		let reply = match request(api::LsModule(sys_id(), path_tok.to_api())).await {
-			Err(api::LsModuleError::TreeUnavailable) =>
+			Err(api::LsModuleError::TreeUnavailable) => {
-				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty."),
+				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty.")
+			},
 			Err(err) => return Err(err),
 			Ok(details) => details,
 		};
@@ -79,10 +80,12 @@ impl ReflMod {
 			return None;
 		}
 		match self.try_populate().await {
-			Err(api::LsModuleError::InvalidPath) =>
+			Err(api::LsModuleError::InvalidPath) => {
-				panic!("Path became invalid since module was created"),
+				panic!("Path became invalid since module was created")
-			Err(api::LsModuleError::IsConstant) =>
+			},
-				panic!("Path previously contained a module but now contains a constant"),
+			Err(api::LsModuleError::IsConstant) => {
+				panic!("Path previously contained a module but now contains a constant")
+			},
 			Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
 			Ok(()) => (),
 		}

orchid-extension/src/stream_reqs.rs

@@ -0,0 +1,72 @@
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::atom::AtomMethod;
+
+/// Represents [std::io::ErrorKind] values that are produced while operating on
+/// already-opened files
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub enum IoErrorKind {
+	BrokenPipe,
+	UnexpectedEof,
+	ConnectionAborted,
+	Other,
+}
+
+/// Represents [std::io::Error] values that are produced while operating on
+/// already-opened files
+#[derive(Clone, Debug, Coding)]
+pub struct IoError {
+	pub message: String,
+	pub kind: IoErrorKind,
+}
+
+/// Read at most the specified number of bytes, but at least one byte, from a
+/// stream. If the returned vector is empty, the stream has reached its end.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+pub struct ReadReq(pub u64);
+impl Request for ReadReq {
+	type Response = Result<Vec<u8>, IoError>;
+}
+impl AtomMethod for ReadReq {
+	const NAME: &str = "orchid::stream::read";
+}
+
+/// Write the specified number of bytes into a stream.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(OutputReq)]
+pub struct WriteReq {
+	pub data: Vec<u8>,
+}
+impl Request for WriteReq {
+	type Response = Result<(), IoError>;
+}
+
+/// Flush a stream, ensuring that all data reached its destination.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(OutputReq)]
+pub struct FlushReq;
+impl Request for FlushReq {
+	type Response = Result<(), IoError>;
+}
+
+/// Close a stream, indicating that no further data will be sent through it.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(OutputReq)]
+pub struct CloseReq;
+impl Request for CloseReq {
+	type Response = Result<(), IoError>;
+}
+
+/// Operations on outbound streams across extension boundaries.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extendable]
+#[allow(clippy::enum_variant_names)]
+pub enum OutputReq {
+	WriteReq(WriteReq),
+	FlushReq(FlushReq),
+	CloseReq(CloseReq),
+}
+impl AtomMethod for OutputReq {
+	const NAME: &str = "orchid::stream::write";
+}

orchid-extension/src/system.rs

@@ -155,7 +155,10 @@ where A: AtomicFeatures {
 	Ok(TAtom { value: *value, untyped: foreign })
 }
 
-pub async fn dep_req<Sys: SystemCard, Req: Request + Into<Sys::Req>>(req: Req) -> Req::Response {
+/// Make a global request to a system that supports this request type. The
+/// target system must either be the system in which this function is called, or
+/// one of its direct dependencies.
+pub async fn sys_req<Sys: SystemCard, Req: Request + Into<Sys::Req>>(req: Req) -> Req::Response {
 	let mut msg = Vec::new();
 	req.into().encode_vec(&mut msg);
 	let cted = cted();

orchid-extension/src/tree.rs

@@ -20,7 +20,7 @@ use crate::api;
 use crate::conv::ToExpr;
 use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
 use crate::func_atom::{ExprFunc, Fun};
-use crate::gen_expr::{GExpr, sym_ref};
+use crate::gen_expr::{GExpr, new_atom, sym_ref};
 
 pub type GenTokTree = TokTree<Expr, GExpr>;
 pub type GenTok = Token<Expr, GExpr>;
@@ -71,12 +71,11 @@ pub fn module(
 	vec![GenMember { name, kind, public, comments: vec![] }]
 }
 pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (String, MemKind) {
-	let kind = MemKind::Mod { members: mems.into_iter().flatten().collect() };
+	(name.to_string(), MemKind::module(mems))
-	(name.to_string(), kind)
 }
 pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
 	let fac =
-		LazyMemberFactory::new(async move |sym| MemKind::Const(Fun::new(sym, xf).await.to_gen().await));
+		LazyMemberFactory::new(async move |sym| MemKind::Const(new_atom(Fun::new(sym, xf).await)));
 	vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
 }
 pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
@@ -113,12 +112,12 @@ pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<Gen
 				let prev = all_members.insert(mem.name.clone(), (unit, mem.comments.into_iter().collect()));
 				assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name);
 			},
-			MemKind::Mod { members } => match all_members.entry(mem.name.clone()) {
+			MemKind::Mod(members) => match all_members.entry(mem.name.clone()) {
 				hashbrown::hash_map::Entry::Vacant(slot) => {
-					slot.insert((MemKind::Mod { members }, mem.comments.into_iter().collect()));
+					slot.insert((MemKind::Mod(members), mem.comments.into_iter().collect()));
 				},
 				hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() {
-					(MemKind::Mod { members: old_items, .. }, old_cmts) => {
+					(MemKind::Mod(old_items), old_cmts) => {
 						let mut swap = vec![];
 						std::mem::swap(&mut swap, old_items);
 						*old_items = merge_trivial([swap, members]);
@@ -167,15 +166,19 @@ impl GenMember {
 
 pub enum MemKind {
 	Const(GExpr),
-	Mod { members: Vec<GenMember> },
+	Mod(Vec<GenMember>),
 	Lazy(LazyMemberFactory),
 }
 impl MemKind {
+	pub async fn cnst(val: impl ToExpr) -> Self { Self::Const(val.to_gen().await) }
+	pub fn module(mems: impl IntoIterator<Item = Vec<GenMember>>) -> Self {
+		Self::Mod(mems.into_iter().flatten().collect())
+	}
 	pub(crate) async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
 		match self {
 			Self::Lazy(lazy) => api::MemberKind::Lazy(add_lazy(ctx, lazy)),
 			Self::Const(c) => api::MemberKind::Const(c.serialize().await),
-			Self::Mod { members } => api::MemberKind::Module(api::Module {
+			Self::Mod(members) => api::MemberKind::Module(api::Module {
 				members: stream(async |mut cx| {
 					for m in members {
 						cx.emit(m.into_api(ctx).await).await

orchid-extension/src/trivial_req.rs

@@ -0,0 +1,28 @@
+use std::io;
+use std::pin::Pin;
+
+use futures::future::LocalBoxFuture;
+use futures::{AsyncRead, AsyncWrite};
+use orchid_base::reqnot::{Receipt, RepWriter, ReqHandle, ReqReader};
+
+pub struct TrivialReqCycle<'a> {
+	pub req: &'a [u8],
+	pub rep: &'a mut Vec<u8>,
+}
+impl<'a> ReqReader<'a> for TrivialReqCycle<'a> {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { Pin::new(&mut self.req) as Pin<&mut _> }
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>> {
+		Box::pin(async { self as Box<_> })
+	}
+}
+impl<'a> ReqHandle<'a> for TrivialReqCycle<'a> {
+	fn start_reply(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>> {
+		Box::pin(async { Ok(self as Box<_>) })
+	}
+}
+impl<'a> RepWriter<'a> for TrivialReqCycle<'a> {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { Pin::new(&mut self.rep) as Pin<&mut _> }
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Receipt<'a>>> {
+		Box::pin(async { Ok(Receipt::_new()) })
+	}
+}

orchid-host/Cargo.toml

@@ -22,10 +22,10 @@ num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
+orchid-extension = { version = "0.1.0", path = "../orchid-extension", optional = true }
 ordered-float = "5.1.0"
 pastey = "0.2.1"
 substack = "1.1.1"
-test_executors = "0.4.1"
 tokio = { version = "1.49.0", features = ["process"], optional = true }
 tokio-util = { version = "0.7.18", features = ["compat"], optional = true }
 trait-set = "0.3.0"
@@ -33,3 +33,4 @@ unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
 
 [features]
 tokio = ["dep:tokio", "dep:tokio-util", "dep:libloading"]
+orchid-extension = ["dep:orchid-extension"]

orchid-host/src/atom.rs

@@ -3,10 +3,14 @@ use std::rc::{Rc, Weak};
 
 use async_once_cell::OnceCell;
 use derive_destructure::destructure;
+#[cfg(feature = "orchid-extension")]
+use orchid_api_traits::{Request, UnderRoot};
 use orchid_base::format::{FmtCtx, FmtUnit, Format, take_first_fmt};
 use orchid_base::location::Pos;
 use orchid_base::reqnot::ClientExt;
 use orchid_base::tree::AtomRepr;
+#[cfg(feature = "orchid-extension")]
+use orchid_extension::atom::AtomMethod;
 
 use crate::api;
 use crate::ctx::Ctx;
@@ -56,6 +60,21 @@ impl AtomHand {
 		Self(Rc::new(AtomData { owner, drop, data, display: OnceCell::new() }))
 	}
 	#[must_use]
+	#[cfg(feature = "orchid-extension")]
+	pub async fn ipc<M: Request + UnderRoot<Root: AtomMethod>>(
+		&self,
+		method: M,
+	) -> Option<M::Response> {
+		use orchid_api_traits::{Decode, Encode};
+		use orchid_base::name::Sym;
+
+		let name = Sym::parse(<M as UnderRoot>::Root::NAME).await.unwrap();
+		let mut buf = Vec::new();
+		method.into_root().encode_vec(&mut buf);
+		let reply_buf = self.req(name.to_api(), buf).await?;
+		Some(M::Response::decode_slice(&mut &reply_buf[..]))
+	}
+	#[must_use]
 	pub async fn call(self, arg: Expr) -> Expr {
 		let owner_sys = self.0.owner.clone();
 		let ctx = owner_sys.ctx();

orchid-host/src/dylib.rs

@@ -2,9 +2,13 @@ use std::io;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
+use futures::io::BufReader;
+use futures::{AsyncBufReadExt, StreamExt};
 use hashbrown::HashMap;
-use libloading::Library;
+use libloading::{Library, Symbol};
 use orchid_base::binary::vt_to_future;
+use orchid_base::logging::log;
+use unsync_pipe::pipe;
 
 use crate::api;
 use crate::ctx::Ctx;
@@ -23,23 +27,16 @@ fn load_dylib(path: &Path) -> Result<Arc<Library>, libloading::Error> {
 	}
 }
 
-#[cfg(feature = "tokio")]
 pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Error> {
-	use futures::io::BufReader;
-	use futures::{AsyncBufReadExt, StreamExt};
-	use libloading::Symbol;
-	use unsync_pipe::pipe;
-
 	let (write_input, input) = pipe(1024);
 	let (output, read_output) = pipe(1024);
-	let (log, read_log) = pipe(1024);
+	let (write_log, read_log) = pipe(1024);
 	let log_path = path.to_string_lossy().to_string();
 	let _ = ctx.spawn(async move {
-		use orchid_base::logging::log;
 		let mut lines = BufReader::new(read_log).lines();
 		while let Some(line) = lines.next().await {
 			match line {
-				Ok(line) => writeln!(log("stderr"), "{log_path} err> {line}").await,
+				Ok(line) => writeln!(log("stderr"), "dylib {log_path} err> {line}").await,
 				Err(e) => match e.kind() {
 					io::ErrorKind::BrokenPipe | io::ErrorKind::UnexpectedEof => break,
 					_ => panic!("Error while reading stderr {e}"),
@@ -56,7 +53,7 @@ pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Err
 		let _ = unsafe { (data as *mut Ctx).as_mut().unwrap().spawn(vt_to_future(vt)) };
 	}
 	let spawner = api::binary::SpawnerBin { data, drop, spawn };
-	let cx = api::binary::ExtensionContext { input, output, log, spawner };
+	let cx = api::binary::ExtensionContext { input, output, log: write_log, spawner };
 	unsafe { (entrypoint)(cx) };
 	Ok(ExtPort { input: Box::pin(write_input), output: Box::pin(read_output) })
 }

orchid-host/src/execute.rs

@@ -110,8 +110,9 @@ impl ExecCtx {
 						(ExprKind::Identity(val.clone()), StackOp::Swap(val))
 					},
 					Err(f) => match &*f.kind().read().await {
-						ExprKind::Arg | ExprKind::Call(..) | ExprKind::Seq(..) | ExprKind::Const(_) =>
+						ExprKind::Arg | ExprKind::Call(..) | ExprKind::Seq(..) | ExprKind::Const(_) => {
-							panic!("This should not appear outside function bodies"),
+							panic!("This should not appear outside function bodies")
+						},
 						ExprKind::Missing => panic!("Should have been replaced"),
 						ExprKind::Atom(a) => {
 							let x_norm = self.unpack_ident(&x).await;

orchid-host/src/expr.rs

@@ -155,8 +155,9 @@ async fn print_exprkind<'a>(
 ) -> FmtUnit {
 	match &ek {
 		ExprKind::Arg => "Arg".to_string().into(),
-		ExprKind::Missing =>
+		ExprKind::Missing => {
-			panic!("This variant is swapped into write guards, so a read can never see it"),
+			panic!("This variant is swapped into write guards, so a read can never see it")
+		},
 		ExprKind::Atom(a) => a.print(c).await,
 		ExprKind::Bottom(e) if e.len() == 1 => format!("Bottom({e})").into(),
 		ExprKind::Bottom(e) => format!("Bottom(\n\t{}\n)", indent(&e.to_string())).into(),

orchid-host/src/extension.rs

@@ -211,12 +211,13 @@ impl Extension {
 													.await
 												{
 													Ok(module) => module,
-													Err(ChildError { kind, .. }) =>
+													Err(ChildError { kind, .. }) => {
 														break 'reply Err(match kind {
 															ChildErrorKind::Private => panic!("Access checking was disabled"),
 															ChildErrorKind::Constant => api::LsModuleError::IsConstant,
 															ChildErrorKind::Missing => api::LsModuleError::InvalidPath,
-														}),
+														});
+													},
 												};
 											let mut members = std::collections::HashMap::new();
 											for (k, v) in &module.members {

orchid-host/src/inline.rs

@@ -0,0 +1,45 @@
+#[cfg(feature = "orchid-extension")]
+use orchid_extension as ox;
+
+#[cfg(feature = "orchid-extension")]
+use crate::ctx::Ctx;
+#[cfg(feature = "orchid-extension")]
+use crate::extension::ExtPort;
+
+#[cfg(feature = "orchid-extension")]
+pub async fn ext_inline(builder: ox::entrypoint::ExtensionBuilder, ctx: Ctx) -> ExtPort {
+	use std::io;
+	use std::rc::Rc;
+
+	use futures::io::BufReader;
+	use futures::{AsyncBufReadExt, StreamExt};
+	use orchid_base::logging::log;
+	use unsync_pipe::pipe;
+
+	let (in_stdin, out_stdin) = pipe(1024);
+	let (in_stdout, out_stdout) = pipe(1024);
+	let (in_stderr, out_stderr) = pipe(1024);
+
+	let name = builder.name;
+
+	std::mem::drop(ctx.spawn(async move {
+		let mut lines = BufReader::new(out_stderr).lines();
+		while let Some(line) = lines.next().await {
+			match line {
+				Ok(line) => writeln!(log("stderr"), "inline {name} err> {line}").await,
+				Err(e) => match e.kind() {
+					io::ErrorKind::BrokenPipe | io::ErrorKind::UnexpectedEof => break,
+					_ => panic!("Error while reading stderr {e}"),
+				},
+			}
+		}
+	}));
+
+	builder.build(ox::ext_port::ExtPort {
+		input: Box::pin(out_stdin),
+		output: Box::pin(in_stdout),
+		log: Box::pin(in_stderr),
+		spawn: Rc::new(move |fut| std::mem::drop(ctx.spawn(fut))),
+	});
+	ExtPort { input: Box::pin(in_stdin), output: Box::pin(out_stdout) }
+}

orchid-host/src/lex.rs

@@ -1,7 +1,10 @@
+use std::collections::VecDeque;
+use std::ops::Range;
+
 use futures::FutureExt;
 use futures::lock::Mutex;
 use orchid_base::clone;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::error::{OrcErrv, OrcRes, mk_errv, report};
 use orchid_base::interner::{IStr, is};
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
@@ -23,10 +26,11 @@ pub struct LexCtx<'a> {
 	pub tail: &'a str,
 	pub sub_trees: &'a mut Vec<Expr>,
 	pub ctx: &'a Ctx,
+	pub produced: &'a mut VecDeque<ParsTokTree>,
 }
 impl<'a> LexCtx<'a> {
 	#[must_use]
-	pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
+	pub fn sub<'b>(&'b mut self, pos: u32, produced: &'b mut VecDeque<ParsTokTree>) -> LexCtx<'b>
 	where 'a: 'b {
 		LexCtx {
 			source: self.source,
@@ -35,6 +39,7 @@ impl<'a> LexCtx<'a> {
 			systems: self.systems,
 			sub_trees: &mut *self.sub_trees,
 			ctx: self.ctx,
+			produced,
 		}
 	}
 	#[must_use]
@@ -44,6 +49,7 @@ impl<'a> LexCtx<'a> {
 	pub fn set_pos(&mut self, pos: u32) { self.tail = &self.source[pos as usize..] }
 	pub fn push_pos(&mut self, delta: u32) { self.set_pos(self.get_pos() + delta) }
 	pub fn set_tail(&mut self, tail: &'a str) { self.tail = tail }
+	pub fn pos_from(&self, tail: &'a str) -> u32 { (self.source.len() - tail.len()) as u32 }
 	#[must_use]
 	pub fn strip_prefix(&mut self, tgt: &str) -> bool {
 		if let Some(src) = self.tail.strip_prefix(tgt) {
@@ -79,23 +85,41 @@ impl<'a> LexCtx<'a> {
 		self.tail = rest;
 		matches
 	}
+	pub fn pop_char(&mut self) -> Option<char> {
+		let mut chars = self.tail.chars();
+		let ret = chars.next()?;
+		self.tail = chars.as_str();
+		Some(ret)
+	}
+	pub fn sr_to(&self, start: u32) -> SrcRange { self.sr(start..self.get_pos()) }
+	pub fn sr(&self, range: Range<u32>) -> SrcRange { SrcRange::new(range, self.path) }
 }
 
-pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
+pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<bool> {
+	ctx.trim(unrep_space);
+	if ctx.tail.is_empty() {
+		return Ok(false);
+	}
 	let start = ctx.get_pos();
-	assert!(
-		!ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space),
-		"Lexing empty string or whitespace to token!\n\
-    Invocations of lex_tok should check for empty string"
-	);
 	let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
 		ParsTok::BR
 	} else if let Some(tail) = (ctx.tail.starts_with(name_start).then_some(ctx.tail))
 		.and_then(|t| t.trim_start_matches(name_char).strip_prefix("::"))
 	{
 		let name = &ctx.tail[..ctx.tail.len() - tail.len() - "::".len()];
-		ctx.set_tail(tail);
+		let mut produced = VecDeque::new();
-		let body = lex_once(ctx).boxed_local().await?;
+		let mut sub_cx = ctx.sub(ctx.pos_from(tail), &mut produced);
+		if !lex_once(&mut sub_cx).boxed_local().await? {
+			return Err(mk_errv(
+				is("Unexpected end of source text").await,
+				":: cannot be the last token",
+				[SrcRange::new(start..ctx.get_pos(), ctx.path)],
+			));
+		}
+		let pos = sub_cx.get_pos();
+		ctx.set_pos(pos);
+		let body = produced.pop_front().expect("lex_once returned true");
+		ctx.produced.extend(produced.into_iter());
 		ParsTok::NS(is(name).await, Box::new(body))
 	} else if ctx.strip_prefix("--[") {
 		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
@@ -113,96 +137,169 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		ParsTok::Comment(is(&tail[2..end]).await)
 	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
 		// fanciness like \$placeh in templates is resolved in the macro engine.
-		ctx.set_tail(tail);
+		let start = ctx.get_pos();
-		let arg = lex_once(ctx).boxed_local().await?;
+		let mut produced = VecDeque::new();
+		let mut sub_cx = ctx.sub(ctx.pos_from(tail), &mut produced);
+		if !lex_once(&mut sub_cx).boxed_local().await? {
+			return Err(mk_errv(
+				is("Unexpected end of file").await,
+				"Expected a lambda argument and body",
+				[SrcRange::new(start..ctx.get_pos(), ctx.path)],
+			));
+		}
+		let pos = sub_cx.get_pos();
+		ctx.set_pos(pos);
+		let arg = produced.pop_front().expect("lex_once returned true");
+		ctx.produced.extend(produced);
 		ctx.trim_ws();
 		ParsTok::LambdaHead(Box::new(arg))
 	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
-		let mut body = Vec::new();
+		let mut body = VecDeque::new();
 		ctx.trim_ws();
 		while !ctx.strip_char(*rp) {
-			if ctx.tail.is_empty() {
+			let mut sub_cx = ctx.sub(ctx.get_pos(), &mut body);
+			if !lex_once(&mut sub_cx).boxed_local().await? {
 				return Err(mk_errv(
 					is("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
 					[SrcRange::new(start..start + 1, ctx.path)],
 				));
 			}
-			body.push(lex_once(ctx).boxed_local().await?);
+			let pos = sub_cx.get_pos();
+			ctx.set_pos(pos);
 			ctx.trim_ws();
 		}
-		ParsTok::S(*paren, body)
+		ParsTok::S(*paren, body.into_iter().collect())
+	} else if let Some(res) = sys_lex(ctx).await {
+		let token = res?;
+		ctx.produced.extend(token);
+		return Ok(true);
+	} else if ctx.tail.starts_with(name_start) {
+		ParsTok::Name(is(ctx.get_start_matches(name_char)).await)
+	} else if ctx.tail.starts_with(op_char) {
+		let whole_tail = ctx.tail;
+		ctx.pop_char().expect("The above check would have failed");
+		let mut tail_after_op = ctx.tail;
+
+		let mut lookahead = Vec::new();
+		while !ctx.tail.is_empty() && ctx.tail.starts_with(op_char) {
+			match sys_lex(ctx).await {
+				None => {
+					ctx.pop_char();
+					tail_after_op = ctx.tail;
+				},
+				Some(sys_res) => {
+					match sys_res {
+						Err(e) => report(e),
+						Ok(tokv) => lookahead = tokv,
+					}
+					break;
+				},
+			}
+		}
+
+		let op_str = &whole_tail[0..whole_tail.len() - tail_after_op.len()];
+		ctx.produced.push_back(ParsTok::Name(is(op_str).await).at(ctx.sr_to(start)));
+		ctx.produced.extend(lookahead);
+		return Ok(true);
 	} else {
-		for sys in ctx.systems {
+		return Err(mk_errv(
-			let mut errors = Vec::new();
+			is("Unrecognized character").await,
-			if ctx.tail.starts_with(|c| sys.can_lex(c)) {
+			"The following syntax is meaningless.",
-				let (source, pos, path) = (ctx.source.clone(), ctx.get_pos(), ctx.path.clone());
+			[SrcRange::new(start..start + 1, ctx.path)],
-				let temp_store = ctx.ctx.exprs.derive();
+		));
-				let ctx_lck = &Mutex::new(&mut *ctx);
+	};
-				let errors_lck = &Mutex::new(&mut errors);
+	ctx.produced.push_back(ParsTokTree { tok, sr: ctx.sr_to(start) });
-				let temp_store_cb = temp_store.clone();
+	Ok(true)
-				let lx = sys
+}
-					.lex(source, path, pos, |pos| {
+
-						clone!(temp_store_cb);
+/// Parse one token via any of the systems, if we can
-						async move {
+///
-							let mut ctx_g = ctx_lck.lock().await;
+/// This function never writes lookahead
-							match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
+pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
-								Ok(t) => Some(api::SubLexed {
+	for sys in ctx.systems {
-									pos: t.sr.end(),
+		let mut errors = Vec::new();
-									tree: ctx_g.ser_subtree(t, temp_store_cb.clone()).await,
+		if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-								}),
+			let (source, pos, path) = (ctx.source.clone(), ctx.get_pos(), ctx.path.clone());
-								Err(e) => {
+			let temp_store = ctx.ctx.exprs.derive();
-									errors_lck.lock().await.push(e);
+			let ctx_lck = &Mutex::new(&mut *ctx);
-									None
+			let errors_lck = &Mutex::new(&mut errors);
-								},
+			let temp_store_cb = temp_store.clone();
-							}
+			let lx = sys
+				.lex(source, path, pos, |pos| {
+					clone!(temp_store_cb);
+					async move {
+						let mut ctx_g = ctx_lck.lock().await;
+						let mut produced = VecDeque::new();
+						let mut sub_cx = ctx_g.sub(pos, &mut produced);
+						let lex_res = lex_once(&mut sub_cx).boxed_local().await;
+						let pos1 = sub_cx.get_pos();
+						ctx_g.set_pos(pos1);
+						match lex_res {
+							Ok(false) => {
+								errors_lck.lock().await.push(mk_errv(
+									is("End of file").await,
+									"Unexpected end of source text",
+									[ctx_g.sr_to(pos)],
+								));
+								None
+							},
+							Ok(true) => {
+								let tok = produced.pop_front().unwrap();
+								Some(api::SubLexed {
+									pos: tok.sr.end(),
+									tree: ctx_g.ser_subtree(tok, temp_store_cb.clone()).await,
+								})
+							},
+							Err(e) => {
+								errors_lck.lock().await.push(e);
+								None
+							},
 						}
-					})
+					}
-					.await;
+				})
-				match lx {
+				.await;
-					Err(e) => return Err(errors.into_iter().fold(OrcErrv::from_api(&e).await, |a, b| a + b)),
+			match lx {
-					Ok(Some(lexed)) => {
+				Err(e) =>
-						ctx.set_pos(lexed.pos);
+					return Some(Err(errors.into_iter().fold(OrcErrv::from_api(&e).await, |a, b| a + b))),
-						let lexed_tree = ctx.des_subtree(&lexed.expr, temp_store).await;
+				Ok(Some(lexed)) => {
-						let stable_tree = recur(lexed_tree, &|tt, r| {
+					ctx.set_pos(lexed.pos);
+					let mut stable_trees = Vec::new();
+					for tok in lexed.expr {
+						stable_trees.push(recur(ctx.des_subtree(&tok, temp_store.clone()).await, &|tt, r| {
 							if let ParsTok::NewExpr(expr) = tt.tok {
 								return ParsTok::Handle(expr).at(tt.sr);
 							}
 							r(tt)
-						});
+						}));
-						return Ok(stable_tree);
+					}
-					},
+					return Some(Ok(stable_trees));
-					Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
+				},
-						Some(errors) => return Err(errors),
+				Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
-						None => continue,
+					Some(errors) => return Some(Err(errors)),
-					},
+					None => continue,
-				}
+				},
 			}
 		}
-		if ctx.tail.starts_with(name_start) {
+	}
-			ParsTok::Name(is(ctx.get_start_matches(name_char)).await)
+	None
-		} else if ctx.tail.starts_with(op_char) {
-			ParsTok::Name(is(ctx.get_start_matches(op_char)).await)
-		} else {
-			return Err(mk_errv(
-				is("Unrecognized character").await,
-				"The following syntax is meaningless.",
-				[SrcRange::new(start..start + 1, ctx.path)],
-			));
-		}
-	};
-	Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
 }
 
 pub async fn lex(text: IStr, path: Sym, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
 	let mut sub_trees = Vec::new();
-	let mut ctx =
+	let mut produced = VecDeque::new();
-		LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };
+	let mut ctx = LexCtx {
-	let mut tokv = Vec::new();
+		source: &text,
+		sub_trees: &mut sub_trees,
+		tail: &text[..],
+		systems,
+		path: &path,
+		ctx,
+		produced: &mut produced,
+	};
 	ctx.trim(unrep_space);
-	while !ctx.tail.is_empty() {
+	while lex_once(&mut ctx).await? {
-		tokv.push(lex_once(&mut ctx).await?);
 		ctx.trim(unrep_space);
 	}
-	Ok(tokv)
+	Ok(produced.into())
 }

orchid-host/src/lib.rs

@@ -3,15 +3,18 @@ use orchid_api as api;
 pub mod atom;
 pub mod ctx;
 pub mod dealias;
+#[cfg(feature = "tokio")]
 pub mod dylib;
 pub mod execute;
 pub mod expr;
 pub mod expr_store;
 pub mod extension;
+pub mod inline;
 pub mod lex;
 pub mod logger;
 pub mod parse;
 pub mod parsed;
+#[cfg(feature = "tokio")]
 pub mod subprocess;
 mod sys_parser;
 pub mod system;

orchid-host/src/subprocess.rs

@@ -3,14 +3,12 @@ use std::{io, process};
 use futures::io::BufReader;
 use futures::{self, AsyncBufReadExt, StreamExt};
 use orchid_base::logging::log;
-#[cfg(feature = "tokio")]
 use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
 
 use crate::ctx::Ctx;
 use crate::extension::ExtPort;
 
-#[cfg(feature = "tokio")]
+pub async fn ext_command(cmd: process::Command, ctx: Ctx) -> io::Result<ExtPort> {
-pub async fn ext_command(cmd: std::process::Command, ctx: Ctx) -> io::Result<ExtPort> {
 	let name = cmd.get_program().to_string_lossy().to_string();
 	let mut child = tokio::process::Command::from(cmd)
 		.stdin(process::Stdio::piped())
@@ -25,9 +23,13 @@ pub async fn ext_command(cmd: std::process::Command, ctx: Ctx) -> io::Result<Ext
 		let _ = child;
 		let mut lines = BufReader::new(child_stderr.compat()).lines();
 		while let Some(line) = lines.next().await {
-			// route stderr with an empty category string. This is not the intended logging
+			match line {
-			// method
+				Ok(line) => writeln!(log("stderr"), "subproc {name} err> {line}").await,
-			writeln!(log("stderr"), "{} err> {}", name, line.expect("Readline implies this")).await;
+				Err(e) => match e.kind() {
+					io::ErrorKind::BrokenPipe | io::ErrorKind::UnexpectedEof => break,
+					_ => panic!("Error while reading stderr {e}"),
+				},
+			}
 		}
 	}));
 	Ok(ExtPort { input: Box::pin(stdin.compat_write()), output: Box::pin(stdout.compat()) })

orchid-host/src/system.rs

@@ -158,14 +158,15 @@ impl System {
 			}
 			match cmod.imports.get(selector) {
 				Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
-				Some(Err(dests)) =>
+				Some(Err(dests)) => {
 					return Err(mk_errv_floating(
 						is("Ambiguous name").await,
 						format!(
 							"{selector} could refer to {}",
 							dests.iter().map(|ri| &ri.target).display("or")
 						),
-					)),
+					));
+				},
 				None => (),
 			}
 			if root_data.root.members.get(selector).is_some() {

orchid-host/src/tree.rs

@@ -63,7 +63,7 @@ impl Root {
 	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym) -> Self {
 		let mut ref_this = self.0.write().await;
 		let this = &mut *ref_this;
-		let mut deferred_consts = HashMap::new();
+		let mut deferred_consts = Vec::new();
 		let consts = this.consts.clone();
 		let mut tfpctx = FromParsedCtx {
 			pars_root: parsed,
@@ -86,7 +86,7 @@ impl Root {
 			.expect("Merge conflict between parsed and existing module");
 		let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
 		*this.ctx.root.write().await = new.downgrade();
-		for (path, (sys_id, pc_id)) in deferred_consts {
+		for (path, sys_id, pc_id) in deferred_consts {
 			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
 			let api_expr = sys.client().request(api::FetchParsedConst(sys.id(), pc_id)).await.unwrap();
 			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), this.ctx.clone()).await;
@@ -383,7 +383,7 @@ pub struct FromParsedCtx<'a> {
 	root: &'a Module,
 	ctx: &'a Ctx,
 	consts: &'a MemoMap<Sym, Expr>,
-	deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
+	deferred_consts: &'a mut Vec<(Sym, api::SysId, api::ParsedConstId)>,
 }
 
 impl Tree for Module {
@@ -435,7 +435,7 @@ impl MemberKind {
 	async fn from_parsed(parsed: &ParsedMemberKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
 		match parsed {
 			ParsedMemberKind::Const(id, sys) => {
-				ctx.deferred_consts.insert(path, (sys.id(), *id));
+				ctx.deferred_consts.push((path, sys.id(), *id));
 				MemberKind::Const
 			},
 			ParsedMemberKind::Mod(m) =>

orchid-std/Cargo.toml

@@ -33,6 +33,7 @@ rust_decimal = "1.39.0"
 subslice-offset = "0.1.1"
 substack = "1.1.1"
 tokio = { version = "1.49.0", features = ["full"] }
+unicode-segmentation = "1.12.0"
 
 [dev-dependencies]
 test_executors = "0.4.1"

orchid-std/src/lib.rs

@@ -1,8 +1,10 @@
+#![allow(refining_impl_trait)]
 mod macros;
 mod std;
 
 pub use std::number::num_atom::{Float, HomoArray, Int, Num};
 pub use std::option::OrcOpt;
+pub use std::protocol::types::{ProtoBuilder, TagBuilder, proto, type_tag};
 pub use std::reflection::sym_atom::{SymAtom, sym_expr};
 pub use std::std_system::StdSystem;
 pub use std::string::str_atom::OrcString;

orchid-std/src/macros/instantiate_tpl.rs

@@ -7,7 +7,7 @@ use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::exec;
 use orchid_extension::expr::Expr;
-use orchid_extension::gen_expr::GExpr;
+use orchid_extension::gen_expr::{GExpr, new_atom};
 
 use crate::macros::mactree::{MacTok, MacTree};
 
@@ -41,7 +41,7 @@ impl OwnedAtom for InstantiateTplCall {
 				Ok(t) => self.argv.push(own(&t).await),
 			};
 			if self.argv.len() < self.argc {
-				return self.to_gen().await;
+				return new_atom(self);
 			}
 			let mut args = self.argv.into_iter();
 			let ret = self.tpl.map(&mut false, &mut |mt| match mt.tok() {
@@ -49,7 +49,7 @@ impl OwnedAtom for InstantiateTplCall {
 				_ => None,
 			});
 			assert!(args.next().is_none(), "Too many arguments for all slots");
-			ret.to_gen().await
+			new_atom(ret)
 		})
 		.await
 		.to_gen()

orchid-std/src/macros/let_line.rs

@@ -11,7 +11,7 @@ use orchid_base::sym;
 use orchid_base::tree::Paren;
 use orchid_extension::atom::TAtom;
 use orchid_extension::conv::TryFromExpr;
-use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
 use orchid_extension::parser::{ConstCtx, PSnippet, PTok, PTokTree, ParsCtx, ParsedLine, Parser};
 
 use crate::macros::mactree::{MacTok, MacTree, MacTreeSeq};
@@ -40,7 +40,7 @@ impl Parser for LetLine {
 		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name, async move |ctx| {
 			let macro_input =
 				MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&aliased, &ctx)).await?).at(sr.pos());
-			Ok(call(sym_ref(sym!(macros::resolve)), [atom(macro_input)]))
+			Ok(call(sym_ref(sym!(macros::resolve)), [new_atom(macro_input)]))
 		})])
 	}
 }

orchid-std/src/macros/macro_lib.rs

@@ -1,9 +1,8 @@
 use orchid_base::sym;
 use orchid_extension::atom::TAtom;
 use orchid_extension::atom_owned::own;
-use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::exec;
-use orchid_extension::gen_expr::{call, sym_ref};
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
 use orchid_extension::tree::{GenMember, fun, prefix};
 
 use crate::macros::mactree::MacTree;
@@ -14,17 +13,36 @@ use crate::{HomoTpl, UntypedTuple};
 pub async fn gen_macro_lib() -> Vec<GenMember> {
 	prefix("macros", [
 		fun(true, "resolve", async |tpl: TAtom<MacTree>| resolve(own(&tpl).await).await),
-		// TODO test whether any of this worked
 		prefix("common", [
-			build_macro(None, ["..", "_"]).finish(),
+			build_macro(None, ["..", "_", "="]).finish(),
 			build_macro(Some(1), ["+"])
-				.rule(mactreev!("...$" lhs 0 macros::common::+ "...$" rhs 1), [async |[lhs, rhs]| {
+				.rule(mactreev!("...$" lhs 1 macros::common::+ "...$" rhs 0), [async |[lhs, rhs]| {
-					call(sym_ref(sym!(std::number::add)), [resolve(lhs).await, resolve(rhs).await])
+					call(sym_ref(sym!(std::ops::add::resolve)), [resolve(lhs).await, resolve(rhs).await])
+				}])
+				.finish(),
+			build_macro(Some(1), ["-"])
+				.rule(mactreev!("...$" lhs 1 macros::common::- "...$" rhs 0), [async |[lhs, rhs]| {
+					call(sym_ref(sym!(std::ops::sub::resolve)), [resolve(lhs).await, resolve(rhs).await])
 				}])
 				.finish(),
 			build_macro(Some(2), ["*"])
-				.rule(mactreev!("...$" lhs 0 macros::common::* "...$" rhs 1), [async |[lhs, rhs]| {
+				.rule(mactreev!("...$" lhs 1 macros::common::* "...$" rhs 0), [async |[lhs, rhs]| {
-					call(sym_ref(sym!(std::number::mul)), [resolve(lhs).await, resolve(rhs).await])
+					call(sym_ref(sym!(std::ops::mul::resolve)), [resolve(lhs).await, resolve(rhs).await])
+				}])
+				.finish(),
+			build_macro(Some(2), ["/"])
+				.rule(mactreev!("...$" lhs 1 macros::common::/ "...$" rhs 0), [async |[lhs, rhs]| {
+					call(sym_ref(sym!(std::ops::div::resolve)), [resolve(lhs).await, resolve(rhs).await])
+				}])
+				.finish(),
+			build_macro(Some(2), ["%"])
+				.rule(mactreev!("...$" lhs 1 macros::common::% "...$" rhs 0), [async |[lhs, rhs]| {
+					call(sym_ref(sym!(std::ops::mod::resolve)), [resolve(lhs).await, resolve(rhs).await])
+				}])
+				.finish(),
+			build_macro(Some(3), ["."])
+				.rule(mactreev!("...$" lhs 1 macros::common::. "...$" rhs 0), [async |[lhs, rhs]| {
+					call(sym_ref(sym!(std::ops::get::resolve)), [resolve(lhs).await, resolve(rhs).await])
 				}])
 				.finish(),
 			build_macro(None, ["comma_list", ","])
@@ -34,14 +52,14 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 						exec(async |mut h| {
 							let recur = resolve(mactree!(macros::common::comma_list "push" tail ;)).await;
 							let mut tail = h.exec::<HomoTpl<TAtom<MacTree>>>(recur).await?;
-							tail.0.insert(0, h.exec(head).await?);
+							tail.0.insert(0, h.exec(new_atom(head)).await?);
 							Ok(tail)
 						})
 						.await
 					}],
 				)
 				.rule(mactreev!(macros::common::comma_list ( "...$" final_tail 0 )), [async |[tail]| {
-					HomoTpl(vec![tail.to_gen().await])
+					HomoTpl(vec![new_atom(tail)])
 				}])
 				.rule(mactreev!(macros::common::comma_list()), [async |[]| UntypedTuple(Vec::new())])
 				.finish(),
@@ -52,14 +70,14 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 						exec(async |mut h| {
 							let recur = resolve(mactree!(macros::common::semi_list "push" tail ;)).await;
 							let mut tail = h.exec::<HomoTpl<TAtom<MacTree>>>(recur).await?;
-							tail.0.insert(0, h.exec(head).await?);
+							tail.0.insert(0, h.exec(new_atom(head)).await?);
 							Ok(tail)
 						})
 						.await
 					}],
 				)
 				.rule(mactreev!(macros::common::semi_list ( "...$" final_tail 0 )), [async |[tail]| {
-					HomoTpl(vec![tail.to_gen().await])
+					HomoTpl(vec![new_atom(tail)])
 				}])
 				.rule(mactreev!(macros::common::semi_list()), [async |[]| UntypedTuple(Vec::new())])
 				.finish(),

orchid-std/src/macros/macro_line.rs

@@ -13,7 +13,7 @@ use orchid_base::tree::{Paren, Token};
 use orchid_base::{clone, sym};
 use orchid_extension::atom::TAtom;
 use orchid_extension::conv::{ToExpr, TryFromExpr};
-use orchid_extension::gen_expr::{call, sym_ref};
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
 use orchid_extension::parser::{PSnippet, ParsCtx, ParsedLine, Parser};
 
 use crate::macros::let_line::{dealias_mac_v, parse_tokv};
@@ -133,7 +133,7 @@ impl Parser for MacroLine {
 				let macro_input =
 					MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&body_mactree, &ctx)).await?)
 						.at(body_sr.pos());
-				Ok(call(sym_ref(sym!(macros::resolve)), [macro_input.to_gen().await]))
+				Ok(call(sym_ref(sym!(macros::resolve)), [new_atom(macro_input)]))
 			}))
 		}
 		let mac_cell = Rc::new(OnceCell::new());
@@ -165,7 +165,7 @@ impl Parser for MacroLine {
 						rules,
 					})))
 				};
-				mac_cell.get_or_init(mac_future).await.clone().to_gen().await
+				mac_cell.get_or_init(mac_future).await.clone().map(new_atom).to_gen().await
 			}))
 		}
 		Ok(lines)

orchid-std/src/macros/macro_system.rs

@@ -18,7 +18,7 @@ use crate::macros::macro_value::Macro;
 use crate::macros::mactree_lexer::MacTreeLexer;
 use crate::macros::match_macros::{MatcherAtom, gen_match_macro_lib};
 use crate::macros::ph_lexer::{PhAtom, PhLexer};
-use crate::macros::std_macros::gen_std_macro_lib;
+use crate::macros::stdlib::gen_std_macro_lib;
 use crate::macros::utils::MacroBodyArgCollector;
 use crate::{MacTree, StdSystem};
 
@@ -55,10 +55,14 @@ impl System for MacroSystem {
 			sym!(macros::common::;),
 			sym!(macros::common::..),
 			sym!(macros::common::_),
+			sym!(macros::common::=),
+			sym!(macros::common::.),
 			sym!(std::tuple::t),
+			sym!(std::record::r),
 			sym!(pattern::match),
 			sym!(pattern::ref),
 			sym!(pattern::=>),
+			sym!(std::fn::[|>]),
 		]
 	}
 	fn lexers() -> Vec<LexerObj> { vec![&MacTreeLexer, &PhLexer] }

orchid-std/src/macros/mactree_lexer.rs

@@ -6,6 +6,7 @@ use orchid_base::error::{OrcRes, mk_errv};
 use orchid_base::interner::is;
 use orchid_base::tokens::PARENS;
 use orchid_base::tree::Paren;
+use orchid_extension::gen_expr::new_atom;
 use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
 use orchid_extension::parser::p_tree2gen;
 use orchid_extension::tree::{GenTok, GenTokTree, x_tok};
@@ -28,11 +29,12 @@ impl Lexer for MacTreeLexer {
 			Ok((tail4, mactree)) => {
 				let range = lctx.pos_tt(tail, tail4);
 				let tok = match &args[..] {
-					[] => x_tok(mactree).await,
+					[] => x_tok(new_atom(mactree)).await,
 					_ => {
 						let instantiate_tpl_call =
 							InstantiateTplCall { argc: args.len(), argv: vec![], tpl: mactree };
-						let call = chain!([x_tok(instantiate_tpl_call).await.at(range.clone())], args);
+						let call =
+							chain!([x_tok(new_atom(instantiate_tpl_call)).await.at(range.clone())], args);
 						GenTok::S(Paren::Round, call.collect())
 					},
 				};

orchid-std/src/macros/match_macros.rs

@@ -15,13 +15,12 @@ use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::{ExecHandle, exec};
 use orchid_extension::expr::{Expr, ExprHandle};
-use orchid_extension::gen_expr::{GExpr, arg, bot, call, lambda, sym_ref};
+use orchid_extension::gen_expr::{GExpr, arg, bot, call, lambda, new_atom, sym_ref};
 use orchid_extension::tree::{GenMember, fun, prefix};
 
 use crate::macros::resolve::resolve;
 use crate::macros::utils::{build_macro, mactree, mactreev};
 use crate::std::reflection::sym_atom::SymAtom;
-use crate::std::tuple::Tuple;
 use crate::{HomoTpl, MacTok, MacTree, OrcOpt, Tpl, UntypedTuple, api};
 
 #[derive(Clone, Coding)]
@@ -83,30 +82,27 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 				.await
 			},
 		),
-		fun(true, "matcher", async |names: HomoTpl<TAtom<SymAtom>>, matcher: Expr| MatcherAtom {
+		fun(true, "matcher", async |names: HomoTpl<TAtom<SymAtom>>, matcher: Expr| {
-			keys: join_all(names.0.iter().map(async |atm| Sym::from_api(atm.0).await)).await,
+			new_atom(MatcherAtom {
-			matcher,
+				keys: join_all(names.0.iter().map(async |atm| Sym::from_api(atm.0).await)).await,
+				matcher,
+			})
 		}),
 		build_macro(None, ["match", "match_rule", "_row", "=>"])
 			.rule(mactreev!("pattern::match" "...$" value 0 { "..$" rules 0 }), [
 				async |[value, rules]| {
 					exec(async move |mut h| {
 						let rule_lines = h
-							.exec::<TAtom<Tuple>>(call(sym_ref(sym!(macros::resolve)), [
+							.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve)), [new_atom(
-								mactree!(macros::common::semi_list "push" rules.clone();).to_gen().await,
+								mactree!(macros::common::semi_list "push" rules.clone();),
-							]))
+							)]))
 							.await?;
 						let mut rule_atoms = Vec::<(TAtom<MatcherAtom>, Expr)>::new();
-						for line_exprh in rule_lines.iter() {
+						for line_mac in rule_lines.0.iter() {
-							let line_mac = h
-								.exec::<TAtom<MacTree>>(Expr::from_handle(
-									ExprHandle::from_ticket(*line_exprh).await,
-								))
-								.await?;
 							let Tpl((matcher, body)) = h
-								.exec(call(sym_ref(sym!(macros::resolve)), [
+								.exec(call(sym_ref(sym!(macros::resolve)), [new_atom(
-									mactree!(pattern::_row "push" own(&line_mac).await ;).to_gen().await,
+									mactree!(pattern::_row "push" own(line_mac).await ;),
-								]))
+								)]))
 								.await?;
 							rule_atoms.push((matcher, body));
 						}
@@ -134,18 +130,18 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 				resolve(mactree!(pattern::match_rule "push" pattern; )).await
 			}])
 			.rule(mactreev!(pattern::match_rule ( macros::common::_ )), [async |[]| {
-				Ok(MatcherAtom {
+				Ok(new_atom(MatcherAtom {
 					keys: Vec::new(),
 					matcher: lambda(0, [OrcOpt(Some(Tpl(()))).to_gen().await]).create().await,
-				})
+				}))
 			}])
 			.rule(mactreev!(pattern::_row ( "...$" pattern 0 pattern::=> "...$" value 1 )), [
 				async |[pattern, mut value]| {
 					exec(async move |mut h| -> OrcRes<Tpl<(TAtom<MatcherAtom>, GExpr)>> {
 						let Ok(pat) = h
-							.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [
+							.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [new_atom(
-								mactree!(pattern::match_rule "push" pattern.clone();).to_gen().await,
+								mactree!(pattern::match_rule "push" pattern.clone();),
-							]))
+							)]))
 							.await
 						else {
 							return Err(mk_errv(
@@ -177,10 +173,10 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 						[name.pos()],
 					));
 				};
-				Ok(MatcherAtom {
+				Ok(new_atom(MatcherAtom {
 					keys: vec![name.clone()],
 					matcher: sym_ref(sym!(pattern::ref_body)).to_expr().await,
-				})
+				}))
 			}])
 			.finish(),
 	])

orchid-std/src/macros/mod.rs

@@ -10,7 +10,7 @@ pub mod match_macros;
 mod ph_lexer;
 mod resolve;
 mod rule;
-pub mod std_macros;
+pub mod stdlib;
 mod utils;
 
 use mactree::{MacTok, MacTree};

orchid-std/src/macros/ph_lexer.rs

@@ -5,6 +5,7 @@ use orchid_base::interner::{es, is};
 use orchid_base::parse::{name_char, name_start};
 use orchid_extension::atom::Atomic;
 use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
+use orchid_extension::gen_expr::new_atom;
 use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
 use orchid_extension::tree::{GenTokTree, x_tok};
 
@@ -72,6 +73,6 @@ impl Lexer for PhLexer {
 			}
 		};
 		let ph_atom = PhAtom(is(name).await.to_api(), phkind);
-		Ok((tail, x_tok(ph_atom).await.at(ctx.pos_tt(line, tail))))
+		Ok((tail, x_tok(new_atom(ph_atom)).await.at(ctx.pos_tt(line, tail))))
 	}
 }

orchid-std/src/macros/resolve.rs

@@ -15,7 +15,7 @@ use orchid_extension::atom::TAtom;
 use orchid_extension::atom_owned::own;
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::{ExecHandle, exec};
-use orchid_extension::gen_expr::{GExpr, arg, bot, call, lambda, sym_ref};
+use orchid_extension::gen_expr::{GExpr, arg, bot, call, lambda, new_atom, sym_ref};
 use orchid_extension::reflection::{ReflMemKind, refl};
 use subslice_offset::SubsliceOffset;
 use substack::Substack;
@@ -283,9 +283,9 @@ async fn mk_body_call(mac: &Macro, rule: &Rule, state: &MatchState<'_>, pos: Pos
 	let mut call_args = vec![];
 	for name in rule.ph_names.iter() {
 		call_args.push(match state.get(name).expect("Missing state entry for placeholder") {
-			StateEntry::Scalar(scal) => (**scal).clone().to_gen().await,
+			StateEntry::Scalar(scal) => new_atom((**scal).clone()),
 			StateEntry::Vec(vec) =>
-				MacTok::S(Paren::Round, MacTreeSeq::new(vec.iter().cloned())).at(Pos::None).to_gen().await,
+				new_atom(MacTok::S(Paren::Round, MacTreeSeq::new(vec.iter().cloned())).at(Pos::None)),
 		});
 	}
 	call(sym_ref(mac.0.module.suffix([rule.body.clone()]).await), call_args).at(pos.clone())

orchid-std/src/macros/rule/build.rs

@@ -123,12 +123,13 @@ async fn mk_scalar(pattern: &MacTree) -> OrcRes<ScalMatcher> {
 			PhKind::Scalar => ScalMatcher::Placeh { key: name.clone() },
 		},
 		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(&body.items).boxed_local().await?)),
-		MacTok::Lambda(..) =>
+		MacTok::Lambda(..) => {
 			return Err(mk_errv(
 				is("Lambda in matcher").await,
 				"Lambdas can't be matched for, only generated in templates",
 				[pattern.pos()],
-			)),
+			));
+		},
 		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
 		MacTok::Bottom(errv) => return Err(errv.clone()),
 	})

orchid-std/src/macros/stdlib/funnctional.rs

@@ -0,0 +1,12 @@
+use orchid_extension::tree::{GenMember, prefix};
+
+use crate::macros::resolve::resolve;
+use crate::macros::utils::{build_macro, mactree, mactreev};
+
+pub async fn gen_functional_macro_lib() -> Vec<GenMember> {
+	prefix("std::fn", [build_macro(Some(4), ["|>"])
+		.rule(mactreev!("...$" lhs 0 "std::fn::|>" "$" fun "...$" rhs 0), [async |[lhs, fun, rhs]| {
+			resolve(mactree!(("push" fun ; "push" lhs ;) "pushv" rhs ;)).await
+		}])
+		.finish()])
+}

orchid-std/src/macros/stdlib/mod.rs

@@ -0,0 +1,20 @@
+pub mod funnctional;
+pub mod option;
+pub mod record;
+pub mod tuple;
+
+use orchid_extension::tree::{GenMember, merge_trivial};
+
+use crate::macros::stdlib::funnctional::gen_functional_macro_lib;
+use crate::macros::stdlib::option::gen_option_macro_lib;
+use crate::macros::stdlib::record::gen_record_macro_lib;
+use crate::macros::stdlib::tuple::gen_tuple_macro_lib;
+
+pub async fn gen_std_macro_lib() -> Vec<GenMember> {
+	merge_trivial([
+		gen_functional_macro_lib().await,
+		gen_option_macro_lib().await,
+		gen_tuple_macro_lib().await,
+		gen_record_macro_lib().await,
+	])
+}

orchid-std/src/macros/stdlib/option.rs

@@ -0,0 +1,56 @@
+use futures::StreamExt;
+use orchid_base::sym;
+use orchid_extension::atom::TAtom;
+use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::exec;
+use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
+use orchid_extension::tree::{GenMember, fun, prefix};
+
+use crate::macros::match_macros::MatcherAtom;
+use crate::macros::resolve::resolve;
+use crate::macros::utils::{build_macro, mactree, mactreev};
+use crate::{OrcOpt, Tpl};
+
+pub async fn gen_option_macro_lib() -> Vec<GenMember> {
+	prefix("std::option", [
+		fun(false, "is_some_body", |sub: TAtom<MatcherAtom>, val: OrcOpt<Expr>| {
+			exec(async move |mut h| {
+				let Some(sub_val) = val.0 else { return Ok(OrcOpt(None)) };
+				sub.run_matcher(&mut h, sub_val).await
+			})
+		}),
+		fun(
+			false,
+			"is_none_body",
+			async |val: OrcOpt<Expr>| {
+				if val.0.is_none() { OrcOpt(Some(Tpl(()))) } else { OrcOpt(None) }
+			},
+		),
+		build_macro(None, ["some", "none"])
+			.rule(mactreev!(pattern::match_rule ( std::option::some "...$" sub_pattern  0)), [
+				|[sub]: [_; _]| {
+					exec(async move |mut h| {
+						let sub = h
+							.exec::<TAtom<MatcherAtom>>(resolve(mactree!(pattern::match_rule "push" sub;)).await)
+							.await?;
+						Ok(new_atom(MatcherAtom {
+							keys: sub.keys().collect().await,
+							matcher: h
+								.register(call(sym_ref(sym!(std::option::is_some_body)), [sub.to_gen().await]))
+								.await,
+						}))
+					})
+				},
+			])
+			.rule(mactreev!(pattern::match_rule(std::option::none)), [|[]: [_; _]| {
+				exec(async |mut h| {
+					Ok(new_atom(MatcherAtom {
+						keys: vec![],
+						matcher: h.register(sym_ref(sym!(std::option::is_none_body))).await,
+					}))
+				})
+			}])
+			.finish(),
+	])
+}

orchid-std/src/macros/stdlib/record.rs

@@ -0,0 +1,45 @@
+use orchid_base::sym;
+use orchid_extension::atom::TAtom;
+use orchid_extension::atom_owned::own;
+use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::exec;
+use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
+use orchid_extension::tree::{GenMember, prefix};
+
+use crate::macros::resolve::resolve;
+use crate::macros::utils::{build_macro, mactree, mactreev};
+use crate::std::string::str_atom::IntStrAtom;
+use crate::{HomoTpl, MacTree, Tpl};
+
+pub async fn gen_record_macro_lib() -> Vec<GenMember> {
+	prefix("std::record", [build_macro(None, ["r", "_row"])
+		.rule(mactreev!(std::record::r[ "...$" elements 0 ]), [async |[elements]: [_; _]| {
+			exec(async move |mut h| {
+				let tup = h
+					.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve)), [new_atom(
+						mactree!((macros::common::comma_list "push" elements ;)),
+					)]))
+					.await?;
+				let mut record = sym_ref(sym!(std::record::empty));
+				for item_exprh in tup.0 {
+					let Tpl((key, value)) = h
+						.exec::<Tpl<(TAtom<IntStrAtom>, Expr)>>(
+							resolve(mactree!(std::record::_row "push" own(&item_exprh).await ;)).await,
+						)
+						.await?;
+					record = call(sym_ref(sym!(std::record::set)), [
+						record.to_gen().await,
+						key.to_gen().await,
+						value.to_gen().await,
+					]);
+				}
+				Ok(record)
+			})
+			.await
+		}])
+		.rule(mactreev!(std::record::_row ( "$" name "...$" value 1 )), [async |[name, value]| {
+			Ok(Tpl((resolve(name).await, resolve(value).await)))
+		}])
+		.finish()])
+}

orchid-std/src/macros/stdlib/tuple.rs

@@ -6,63 +6,21 @@ use orchid_extension::atom_owned::own;
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::exec;
 use orchid_extension::expr::Expr;
-use orchid_extension::gen_expr::{GExpr, call, sym_ref};
+use orchid_extension::gen_expr::{GExpr, call, new_atom, sym_ref};
 use orchid_extension::tree::{GenMember, fun, prefix};
 
 use crate::macros::match_macros::MatcherAtom;
-use crate::macros::resolve::resolve;
 use crate::macros::utils::{build_macro, mactree, mactreev};
-use crate::{HomoTpl, MacTree, OrcOpt, Tpl};
+use crate::{HomoTpl, MacTree, OrcOpt};
 
-pub async fn gen_std_macro_lib() -> Vec<GenMember> {
+pub async fn gen_tuple_macro_lib() -> Vec<GenMember> {
-	prefix("std", [
+	prefix("std::tuple", [
-		prefix("option", [
-			fun(false, "is_some_body", |sub: TAtom<MatcherAtom>, val: OrcOpt<Expr>| {
-				exec(async move |mut h| {
-					let Some(sub_val) = val.0 else { return Ok(OrcOpt(None)) };
-					sub.run_matcher(&mut h, sub_val).await
-				})
-			}),
-			fun(false, "is_none_body", async |val: OrcOpt<Expr>| {
-				if val.0.is_none() { OrcOpt(Some(Tpl(()))) } else { OrcOpt(None) }
-			}),
-			build_macro(None, ["some", "none"])
-				.rule(mactreev!(pattern::match_rule ( std::option::some "...$" sub_pattern  0)), [
-					|[sub]: [_; _]| {
-						exec(async move |mut h| {
-							let sub = h
-								.exec::<TAtom<MatcherAtom>>(
-									resolve(mactree!(pattern::match_rule "push" sub;)).await,
-								)
-								.await?;
-							Ok(MatcherAtom {
-								keys: sub.keys().collect().await,
-								matcher: h
-									.register(call(sym_ref(sym!(std::option::is_some_body)), [sub
-										.to_gen()
-										.await]))
-									.await,
-							})
-						})
-					},
-				])
-				.rule(mactreev!(pattern::match_rule(std::option::none)), [|[]: [_; _]| {
-					exec(async |mut h| {
-						Ok(MatcherAtom {
-							keys: vec![],
-							matcher: h.register(sym_ref(sym!(std::option::is_none_body))).await,
-						})
-					})
-				}])
-				.finish(),
-		]),
-		prefix("tuple", [
 			build_macro(None, ["t"])
 				.rule(mactreev!(std::tuple::t [ "...$" elements 0 ]), [|[elements]: [_; _]| {
 					exec(async move |mut h| {
 						let tup = h
 							.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve)), [
-								mactree!((macros::common::comma_list "push" elements ;)).to_gen().await,
+								new_atom(mactree!((macros::common::comma_list "push" elements ;))),
 							]))
 							.await?;
 						let val = stream::iter(&tup.0[..])
@@ -94,37 +52,36 @@ pub async fn gen_std_macro_lib() -> Vec<GenMember> {
 				])
 				.finish(),
 			fun(false, "matcher_body", tuple_matcher_body),
-		]),
+		])
-	])
 }
 
 fn parse_tpl(elements: MacTree, tail_matcher: Option<MacTree>) -> impl Future<Output = GExpr> {
-	exec(async move |mut h| -> OrcRes<MatcherAtom> {
+	exec(async move |mut h| -> OrcRes<GExpr> {
 		let tup = h
-			.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve)), [
+			.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve)), [new_atom(
-				mactree!((macros::common::comma_list "push" elements ;)).to_gen().await,
+				mactree!((macros::common::comma_list "push" elements ;)),
-			]))
+			)]))
 			.await?;
 		let mut subs = Vec::with_capacity(tup.0.len());
 		for mac_a in &tup.0[..] {
 			let mac = own(mac_a).await;
 			let sub = h
-				.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [
+				.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [new_atom(
-					mactree!(pattern::match_rule ("push" mac ;)).to_gen().await,
+					mactree!(pattern::match_rule ("push" mac ;)),
-				]))
+				)]))
 				.await?;
 			subs.push(sub);
 		}
 		let tail_matcher = match tail_matcher {
 			Some(mac) => Some(
-				h.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [
+				h.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [new_atom(
-					mactree!(pattern::match_rule "push" mac ;).to_gen().await,
+					mactree!(pattern::match_rule "push" mac ;),
-				]))
+				)]))
 				.await?,
 			),
 			None => None,
 		};
-		Ok(MatcherAtom {
+		Ok(new_atom(MatcherAtom {
 			keys: stream::iter(&subs[..])
 				.flat_map(|t| t.keys())
 				.chain(stream::iter(&tail_matcher).flat_map(|mat| mat.keys()))
@@ -136,7 +93,7 @@ fn parse_tpl(elements: MacTree, tail_matcher: Option<MacTree>) -> impl Future<Ou
 			])
 			.to_expr()
 			.await,
-		})
+		}))
 	})
 }
 

orchid-std/src/macros/utils.rs

@@ -11,7 +11,7 @@ use orchid_base::name::{NameLike, Sym, VPath};
 use orchid_extension::atom::{Atomic, TAtom};
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
 use orchid_extension::conv::ToExpr;
-use orchid_extension::gen_expr::{GExpr, sym_ref};
+use orchid_extension::gen_expr::{GExpr, new_atom, sym_ref};
 use orchid_extension::tree::{GenMember, MemKind, cnst, lazy};
 
 use crate::macros::macro_value::{Macro, MacroData, Rule};
@@ -48,7 +48,7 @@ impl OwnedAtom for MacroBodyArgCollector {
 		if self.argc == self.args.len() {
 			(self.cb)(self.args).await.to_gen().await
 		} else {
-			self.to_gen().await
+			new_atom(self)
 		}
 	}
 }
@@ -86,15 +86,20 @@ impl MacroBuilder {
 				let argv = [].into_iter().collect_array().expect("N is 0");
 				MemKind::Const(body(argv).await.to_gen().await)
 			}),
-			1.. => cnst(true, name, MacroBodyArgCollector {
+			1.. => cnst(
-				argc: N,
+				true,
-				args: Vec::new(),
+				name,
-				cb: Rc::new(move |argv| {
+				new_atom(MacroBodyArgCollector {
-					let arr = argv.into_iter().collect_array::<N>().expect("argc should enforce the length");
+					argc: N,
-					let body = body.clone();
+					args: Vec::new(),
-					Box::pin(async move { body(arr).await.to_gen().await })
+					cb: Rc::new(move |argv| {
+						let arr =
+							argv.into_iter().collect_array::<N>().expect("argc should enforce the length");
+						let body = body.clone();
+						Box::pin(async move { body(arr).await.to_gen().await })
+					}),
 				}),
-			}),
+			),
 		});
 		self.patterns.push(pat);
 		self
@@ -105,35 +110,31 @@ impl MacroBuilder {
 		let main_const = lazy(true, &format!("__macro__{name}"), async move |path| {
 			let module = (Sym::new(path.split_last_seg().1.iter().cloned()).await)
 				.expect("Default macro in global root");
-			MemKind::Const(
+			MemKind::Const(new_atom(Macro(Rc::new(MacroData {
-				Macro(Rc::new(MacroData {
+				canonical_name: module.suffix([is(name).await]).await,
-					canonical_name: module.suffix([is(name).await]).await,
+				module,
-					module,
+				prio,
-					prio,
+				rules: stream(async |mut h| {
-					rules: stream(async |mut h| {
+					for (counter, pattern) in patterns.into_iter().enumerate() {
-						for (counter, pattern) in patterns.into_iter().enumerate() {
+						let mut placeholders = Vec::new();
-							let mut placeholders = Vec::new();
+						pattern.map(&mut false, &mut |tt| {
-							pattern.map(&mut false, &mut |tt| {
+							if let MacTok::Ph(ph) = &*tt.tok {
-								if let MacTok::Ph(ph) = &*tt.tok {
+								placeholders.push(ph.name.clone())
-									placeholders.push(ph.name.clone())
+							}
-								}
+							None
-								None
+						});
-							});
+						h.emit(Rule {
-							h.emit(Rule {
+							matcher: Matcher::new(pattern.clone()).await.unwrap(),
-								matcher: Matcher::new(pattern.clone()).await.unwrap(),
+							pattern,
-								pattern,
+							ph_names: placeholders,
-								ph_names: placeholders,
+							body: is(&format!("({name})::{counter}")).await,
-								body: is(&format!("({name})::{counter}")).await,
+						})
-							})
+						.await;
-							.await;
+					}
-						}
+				})
-					})
+				.collect()
-					.collect()
-					.await,
-				}))
-				.to_gen()
 				.await,
-			)
+			}))))
 		});
 		let kw_consts = own_kws[1..].iter().flat_map(|kw| {
 			lazy(true, &format!("__macro__{kw}"), async move |path| {
@@ -188,6 +189,15 @@ macro_rules! mactreev_impl {
 		$ret.push($arg);
 		$crate::macros::utils::mactreev_impl!(@RECUR $ret $($tail)*);
 	};
+	(@RECUR $ret:ident "pushv" $arg:expr ; $($tail:tt)*) => {
+		let $crate::macros::mactree::MacTok::S(_, body) = $arg.tok() else {
+			panic!("pushv used with non-vec value")
+		};
+		for item in body.items.iter() {
+			$ret.push(item.clone());
+		}
+		$crate::macros::utils::mactreev_impl!(@RECUR $ret $($tail)*);
+	};
 	(@RECUR $ret:ident "l_" $arg:expr ; ($($body:tt)*) $($tail:tt)*) => {
 		$ret.push(MacTok::Lambda(
 			MacTok::Name($arg).at(orchid_base::location::Pos::Inherit),

orchid-std/src/std/binary/binary_atom.rs

@@ -0,0 +1,25 @@
+use std::borrow::Cow;
+use std::pin::Pin;
+use std::rc::Rc;
+
+use futures::AsyncWrite;
+use orchid_api_traits::Encode;
+use orchid_extension::atom::Atomic;
+use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
+
+#[derive(Clone)]
+pub struct BlobAtom(pub(crate) Rc<Vec<u8>>);
+impl Atomic for BlobAtom {
+	type Variant = OwnedVariant;
+	type Data = ();
+}
+impl OwnedAtom for BlobAtom {
+	type Refs = ();
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	async fn serialize(&self, write: Pin<&mut (impl AsyncWrite + ?Sized)>) -> Self::Refs {
+		self.0.encode(write).await.unwrap()
+	}
+	async fn deserialize(mut dctx: impl DeserializeCtx, _: Self::Refs) -> Self {
+		Self(dctx.read::<Rc<Vec<u8>>>().await)
+	}
+}

orchid-std/src/std/binary/binary_lib.rs

@@ -0,0 +1,146 @@
+use std::rc::Rc;
+
+use orchid_base::error::{OrcErrv, mk_errv};
+use orchid_base::interner::is;
+use orchid_extension::atom::TAtom;
+use orchid_extension::atom_owned::own;
+use orchid_extension::func_atom::get_arg_posv;
+use orchid_extension::gen_expr::new_atom;
+use orchid_extension::tree::{GenMember, comments, fun, prefix};
+
+use crate::std::binary::binary_atom::BlobAtom;
+use crate::std::boolean::Bool;
+use crate::{Int, OrcOpt, Tpl};
+
+async fn bounds_error(
+	expected: String,
+	blob: &BlobAtom,
+	args: impl IntoIterator<Item = usize>,
+) -> OrcErrv {
+	mk_errv(
+		is("Index out of bounds").await,
+		format!("Selected {expected} from blob of len {}", blob.0.len()),
+		get_arg_posv(args).await,
+	)
+}
+
+pub fn gen_binary_lib() -> Vec<GenMember> {
+	prefix("std", [comments(
+		["A Blob is a sequence of bytes stored and processed efficiently."],
+		prefix("binary", [
+			comments(
+				["Appends a binary blob to another", "|type: Blob -> Blob -> Blob|"],
+				fun(true, "concat", async |a: TAtom<BlobAtom>, b: TAtom<BlobAtom>| {
+					new_atom(BlobAtom(Rc::new(
+						own(&a).await.0.iter().chain(&own(&b).await.0[..]).copied().collect(),
+					)))
+				}),
+			),
+			comments(
+				[
+					"Copies out a subsection of the binary into a new blob \
+					specified by starting point and length",
+					"|type: Blob -> Int -> Int -> Blob|",
+				],
+				fun(true, "slice", async |a: TAtom<BlobAtom>, Int(start): Int, Int(len): Int| {
+					let blob = own(&a).await;
+					if start + len > blob.0.len() as i64 {
+						return Err(bounds_error(format!("{start}+{len}"), &blob, 0..3).await);
+					}
+					let sub = blob.0[start as usize..(start + len) as usize].to_vec();
+					Ok(new_atom(BlobAtom(Rc::new(sub))))
+				}),
+			),
+			comments(
+				[
+					"Return the index where the second binary appears as a subsection of the first",
+					"|type: Blob -> Blob -> std::option Int|",
+				],
+				fun(true, "find", async |haystack: TAtom<BlobAtom>, needle: TAtom<BlobAtom>| {
+					let haystack_vec = own(&haystack).await;
+					let needle_vec = own(&needle).await;
+					for i in 0..haystack_vec.0.len() - needle_vec.0.len() {
+						if haystack_vec.0[i..].starts_with(&needle_vec.0) {
+							return OrcOpt(Some(Int(i as i64)));
+						}
+					}
+					OrcOpt(None)
+				}),
+			),
+			comments(
+				[
+					"Splits the binary into two halves at the given byte index",
+					"|type: Blob -> Int -> std::tuple Blob Blob|",
+				],
+				fun(true, "split", async |a: TAtom<BlobAtom>, i: Int| {
+					let v = own(&a).await;
+					if v.0.len() < i.0 as usize {
+						return Err(bounds_error(i.0.to_string(), &v, 1..2).await);
+					}
+					let (l, r) = v.0.split_at(i.0 as usize);
+					Ok(Tpl((
+						new_atom(BlobAtom(Rc::new(l.to_vec()))),
+						new_atom(BlobAtom(Rc::new(r.to_vec()))),
+					)))
+				}),
+			),
+			comments(
+				[
+					"Takes a binary, a starting point, a length no greater than 8, and a boolean flag \
+					which is true if the number is little endian. Reads a usize from \
+					the specified location in the binary.",
+					"|type: Blob -> Int -> Int -> Bool -> Int|",
+				],
+				fun(
+					true,
+					"get_int",
+					async |bin: TAtom<BlobAtom>, Int(start): Int, Int(len): Int, Bool(le): Bool| {
+						let vec = own(&bin).await;
+						if start + len > vec.0.len() as i64 {
+							return Err(bounds_error(format!("{start}+{len}"), &vec, 1..3).await);
+						}
+						if 8 < len {
+							return Err(mk_errv(
+								is("Too many bytes for int conversion").await,
+								format!("At most 8 bytes fit into an Int, requested {len}"),
+								get_arg_posv(3..4).await,
+							));
+						}
+						let slice = &vec.0[start as usize..(start + len) as usize];
+						let mut data = [0u8; 8];
+						Ok(Int(if le {
+							data[..len as usize].copy_from_slice(slice);
+							i64::from_le_bytes(data)
+						} else {
+							data[(8 - len as usize)..].copy_from_slice(slice);
+							i64::from_be_bytes(data)
+						}))
+					},
+				),
+			),
+			comments(
+				[
+					"Takes a length no greater than int_bytes, a little endian flag and a number to encode. \
+					Turns the least significant bytes of the given int into a binary.",
+					"|type: Int -> Bool -> Int -> Blob|",
+				],
+				fun(true, "from_num", async |len: Int, le: Bool, val: Int| {
+					if 8 < len.0 {
+						return Err(mk_errv(
+							is("Too many bytes for int conversion").await,
+							format!("Ints are 8 bytes, attempted to write {} byte buffer", len.0),
+							get_arg_posv(0..1).await,
+						));
+					}
+					let data = if le.0 { val.0.to_le_bytes() } else { val.0.to_be_bytes() };
+					let data = if le.0 { &data[..len.0 as usize] } else { &data[(8 - len.0 as usize)..] };
+					Ok(new_atom(BlobAtom(Rc::new(data.to_vec()))))
+				}),
+			),
+			comments(
+				["Returns the number of bytes in a binary", "|type: Blob -> Int|"],
+				fun(true, "size", async |blob: TAtom<BlobAtom>| Int(own(&blob).await.0.len() as i64)),
+			),
+		]),
+	)])
+}

orchid-std/src/std/binary/mod.rs

@@ -0,0 +1,2 @@
+pub mod binary_atom;
+pub mod binary_lib;

orchid-std/src/std/boolean.rs

@@ -0,0 +1,47 @@
+use orchid_api_derive::Coding;
+use orchid_base::error::OrcRes;
+use orchid_base::format::FmtUnit;
+use orchid_base::sym;
+use orchid_extension::atom::{Atomic, TAtom};
+use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::{GExpr, sym_ref};
+use orchid_extension::tree::{GenMember, cnst, comments, fun, prefix};
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct Bool(pub bool);
+impl Atomic for Bool {
+	type Variant = ThinVariant;
+	type Data = Self;
+}
+impl ThinAtom for Bool {
+	async fn print(&self) -> FmtUnit { self.0.to_string().into() }
+}
+impl TryFromExpr for Bool {
+	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
+		match TAtom::<Bool>::downcast(expr.handle()).await {
+			Err(e) => Err(e.mk_err().await),
+			Ok(atom) => Ok(atom.value),
+		}
+	}
+}
+impl ToExpr for Bool {
+	async fn to_gen(self) -> GExpr {
+		sym_ref(if self.0 { sym!(std::true) } else { sym!(std::false) })
+	}
+}
+
+pub fn gen_bool_lib() -> Vec<GenMember> {
+	prefix("std", [
+		comments(
+			[
+				"Returns the second argument if the bool is true, the third argument otherwise",
+				"|type: Bool -> T -> T -> T|",
+			],
+			fun(true, "ifthenelse", async |Bool(b): Bool, t: Expr, f: Expr| if b { t } else { f }),
+		),
+		cnst(true, "true", Bool(true)),
+		cnst(true, "false", Bool(false)),
+	])
+}

orchid-std/src/std/mod.rs

@@ -1,4 +1,7 @@
+pub mod binary;
+pub mod boolean;
 pub mod number;
+pub mod ops;
 pub mod option;
 pub mod protocol;
 pub mod record;

orchid-std/src/std/number/num_atom.rs

@@ -1,24 +1,42 @@
-use orchid_api_derive::Coding;
+use std::io;
+
+use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
 use orchid_base::name::Sym;
 use orchid_base::number::Numeric;
-use orchid_extension::atom::{AtomFactory, Atomic, AtomicFeatures, Supports, TAtom, ToAtom};
+use orchid_base::reqnot::{Receipt, ReqHandle, ReqHandleExt};
+use orchid_base::sym;
+use orchid_extension::atom::{Atomic, MethodSetBuilder, Supports, TAtom};
 use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
-use orchid_extension::conv::TryFromExpr;
+use orchid_extension::conv::{ToExpr, TryFromExpr};
 use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::sym_ref;
+use orchid_extension::system::sys_req;
 use ordered_float::NotNan;
 use rust_decimal::prelude::Zero;
 
-use crate::std::protocol::types::GetTagIdMethod;
+use crate::std::protocol::types::{GetImpl, ProtocolMethod};
+use crate::std::std_system::StdReq;
 use crate::std::string::to_string::ToStringMethod;
+use crate::{StdSystem, api};
 
-#[derive(Clone, Debug, Coding)]
+#[derive(Debug, Clone, Copy, Coding, Hash, PartialEq, Eq, Hierarchy)]
+#[extends(StdReq)]
+pub struct CreateInt(pub Int);
+impl Request for CreateInt {
+	type Response = api::ExprTicket;
+}
+
+#[derive(Clone, Copy, Debug, Coding, Hash, PartialEq, Eq)]
 pub struct Int(pub i64);
 impl Atomic for Int {
 	type Variant = ThinVariant;
 	type Data = Self;
+	fn reg_reqs() -> MethodSetBuilder<Self> {
+		MethodSetBuilder::new().handle::<ProtocolMethod>().handle::<ToStringMethod>()
+	}
 }
 impl ThinAtom for Int {
 	async fn print(&self) -> FmtUnit { self.0.to_string().into() }
@@ -28,22 +46,64 @@ impl TryFromExpr for Int {
 		TAtom::<Int>::try_from_expr(expr).await.map(|t| t.value)
 	}
 }
-impl Supports<GetTagIdMethod> for Int {
+impl ToExpr for Int {
-	async fn handle(&self, _: GetTagIdMethod) -> <GetTagIdMethod as Request>::Response {
+	async fn to_gen(self) -> orchid_extension::gen_expr::GExpr {
-		Sym::parse("std::number::Int").await.unwrap().to_api()
+		Expr::deserialize(sys_req::<StdSystem, _>(CreateInt(self)).await).await.to_gen().await
+	}
+}
+impl Supports<ProtocolMethod> for Int {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> io::Result<Receipt<'a>> {
+		match req {
+			ProtocolMethod::GetTagId(req) => hand.reply(&req, &sym!(std::number::Int).to_api()).await,
+			ProtocolMethod::GetImpl(ref req @ GetImpl(key)) => {
+				let name = Sym::from_api(key).await;
+				let val = if name == sym!(std::ops::add) {
+					sym_ref(sym!(std::number::add))
+				} else if name == sym!(std::ops::sub) {
+					sym_ref(sym!(std::number::sub))
+				} else if name == sym!(std::ops::mul) {
+					sym_ref(sym!(std::number::mul))
+				} else if name == sym!(std::ops::div) {
+					sym_ref(sym!(std::number::idiv))
+				} else if name == sym!(std::ops::mod) {
+					sym_ref(sym!(std::number::imod))
+				} else {
+					return hand.reply(req, &None).await;
+				};
+				hand.reply(req, &Some(val.create().await.serialize().await)).await
+			},
+		}
 	}
 }
 impl Supports<ToStringMethod> for Int {
-	async fn handle(&self, _: ToStringMethod) -> <ToStringMethod as Request>::Response {
+	async fn handle<'a>(
-		self.0.to_string()
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ToStringMethod,
+	) -> io::Result<Receipt<'a>> {
+		hand.reply(&req, &self.0.to_string()).await
 	}
 }
 
-#[derive(Clone, Debug, Coding)]
+#[derive(Clone, Copy, Debug, Coding, Hash, PartialEq, Eq, Hierarchy)]
+#[extends(StdReq)]
+pub struct CreateFloat(pub Float);
+impl Request for CreateFloat {
+	type Response = api::ExprTicket;
+}
+
+#[derive(Clone, Copy, Debug, Coding, Hash, PartialEq, Eq)]
 pub struct Float(pub NotNan<f64>);
 impl Atomic for Float {
 	type Variant = ThinVariant;
 	type Data = Self;
+	fn reg_reqs() -> MethodSetBuilder<Self> {
+		MethodSetBuilder::new().handle::<ProtocolMethod>().handle::<ToStringMethod>()
+	}
 }
 impl ThinAtom for Float {
 	async fn print(&self) -> FmtUnit { self.0.to_string().into() }
@@ -53,9 +113,46 @@ impl TryFromExpr for Float {
 		Ok(Self(Num::try_from_expr(expr).await?.0.to_f64()))
 	}
 }
+impl ToExpr for Float {
+	async fn to_gen(self) -> orchid_extension::gen_expr::GExpr {
+		Expr::deserialize(sys_req::<StdSystem, _>(CreateFloat(self)).await).await.to_gen().await
+	}
+}
+impl Supports<ProtocolMethod> for Float {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> io::Result<Receipt<'a>> {
+		match req {
+			ProtocolMethod::GetTagId(req) => hand.reply(&req, &sym!(std::number::Float).to_api()).await,
+			ProtocolMethod::GetImpl(ref req @ GetImpl(key)) => {
+				let name = Sym::from_api(key).await;
+				let val = if name == sym!(std::ops::add) {
+					sym_ref(sym!(std::number::add))
+				} else if name == sym!(std::ops::sub) {
+					sym_ref(sym!(std::number::sub))
+				} else if name == sym!(std::ops::mul) {
+					sym_ref(sym!(std::number::mul))
+				} else if name == sym!(std::ops::div) {
+					sym_ref(sym!(std::number::fdiv))
+				} else if name == sym!(std::ops::mod) {
+					sym_ref(sym!(std::number::fmod))
+				} else {
+					return hand.reply(req, &None).await;
+				};
+				hand.reply(req, &Some(val.create().await.serialize().await)).await
+			},
+		}
+	}
+}
 impl Supports<ToStringMethod> for Float {
-	async fn handle(&self, _: ToStringMethod) -> <ToStringMethod as Request>::Response {
+	async fn handle<'a>(
-		self.0.to_string()
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ToStringMethod,
+	) -> io::Result<Receipt<'a>> {
+		hand.reply(&req, &self.0.to_string()).await
 	}
 }
 
@@ -72,12 +169,14 @@ impl TryFromExpr for Num {
 		}
 	}
 }
-impl ToAtom for Num {
+impl ToExpr for Num {
-	fn to_atom_factory(self) -> AtomFactory {
+	async fn to_gen(self) -> orchid_extension::gen_expr::GExpr {
 		match self.0 {
-			Numeric::Float(f) => Float(f).factory(),
+			Numeric::Float(f) => Float(f).to_expr().await,
-			Numeric::Int(i) => Int(i).factory(),
+			Numeric::Int(i) => Int(i).to_expr().await,
 		}
+		.to_gen()
+		.await
 	}
 }
 

orchid-std/src/std/number/num_lexer.rs

@@ -2,7 +2,7 @@ use std::ops::RangeInclusive;
 
 use orchid_base::error::OrcRes;
 use orchid_base::number::{num_to_errv, parse_num};
-use orchid_extension::atom::ToAtom;
+use orchid_extension::conv::ToExpr;
 use orchid_extension::lexer::{LexContext, Lexer};
 use orchid_extension::tree::{GenTokTree, x_tok};
 
@@ -15,10 +15,10 @@ impl Lexer for NumLexer {
 	async fn lex<'a>(all: &'a str, lxcx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
 		let ends_at = all.find(|c: char| !c.is_ascii_hexdigit() && !"xX._pP".contains(c));
 		let (chars, tail) = all.split_at(ends_at.unwrap_or(all.len()));
-		let fac = match parse_num(chars) {
+		match parse_num(chars) {
-			Ok(numeric) => Num(numeric).to_atom_factory(),
+			Ok(numeric) =>
-			Err(e) => return Err(num_to_errv(e, lxcx.pos(all), lxcx.src()).await),
+				Ok((tail, x_tok(Num(numeric).to_gen().await).await.at(lxcx.pos_lt(chars.len(), tail)))),
-		};
+			Err(e) => Err(num_to_errv(e, lxcx.pos(all), lxcx.src()).await),
-		Ok((tail, x_tok(fac).await.at(lxcx.pos_lt(chars.len(), tail))))
+		}
 	}
 }

orchid-std/src/std/number/num_lib.rs

@@ -1,34 +1,60 @@
+use orchid_base::error::mk_errv;
+use orchid_base::interner::is;
 use orchid_base::number::Numeric;
+use orchid_extension::func_atom::get_arg;
 use orchid_extension::tree::{GenMember, fun, prefix};
 use ordered_float::NotNan;
+use rust_decimal::prelude::ToPrimitive;
 
 use super::num_atom::{Float, HomoArray, Int, Num};
 
 pub fn gen_num_lib() -> Vec<GenMember> {
 	prefix("std::number", [
-		fun(true, "add", async |a: Num, b: Num| {
+		fun(false, "add", async |a: Num, b: Num| {
 			Num(match HomoArray::new([a.0, b.0]) {
 				HomoArray::Int([a, b]) => Numeric::Int(a + b),
 				HomoArray::Float([a, b]) => Numeric::Float(a + b),
 			})
 		}),
-		fun(true, "neg", async |a: Num| {
+		fun(false, "sub", async |a: Num, b: Num| {
+			Num(match HomoArray::new([a.0, b.0]) {
+				HomoArray::Int([a, b]) => Numeric::Int(a - b),
+				HomoArray::Float([a, b]) => Numeric::Float(a - b),
+			})
+		}),
+		fun(false, "neg", async |a: Num| {
 			Num(match a.0 {
 				Numeric::Int(i) => Numeric::Int(-i),
 				Numeric::Float(f) => Numeric::Float(-f),
 			})
 		}),
-		fun(true, "mul", async |a: Num, b: Num| {
+		fun(false, "mul", async |a: Num, b: Num| {
 			Num(match HomoArray::new([a.0, b.0]) {
 				HomoArray::Int([a, b]) => Numeric::Int(a * b),
 				HomoArray::Float([a, b]) => Numeric::Float(a * b),
 			})
 		}),
-		fun(true, "idiv", async |a: Int, b: Int| Int(a.0 / b.0)),
+		fun(false, "idiv", async |a: Int, b: Int| Int(a.0 / b.0)),
-		fun(true, "imod", async |a: Int, b: Int| Int(a.0 % b.0)),
+		fun(false, "imod", async |a: Int, b: Int| Int(a.0 % b.0)),
-		fun(true, "fdiv", async |a: Float, b: Float| Float(a.0 / b.0)),
+		fun(false, "fdiv", async |a: Float, b: Float| Float(a.0 / b.0)),
-		fun(true, "fmod", async |a: Float, b: Float| {
+		fun(false, "fmod", async |a: Float, b: Float| {
 			Float(a.0 - NotNan::new((a.0 / b.0).trunc()).unwrap() * b.0)
 		}),
+		fun(false, "to_i", async |a: Num| {
+			Ok(Int(match a.0 {
+				Numeric::Int(i) => i,
+				Numeric::Float(f) => match f.to_i64() {
+					Some(i) => i,
+					None => {
+						return Err(mk_errv(
+							is("Float out of range").await,
+							format!("{f} is not representable as an integer"),
+							[get_arg(0).pos().await],
+						));
+					},
+				},
+			}))
+		}),
+		fun(false, "to_f", async |a: Num| Float(a.0.to_f64())),
 	])
 }

orchid-std/src/std/ops/mod.rs

@@ -0,0 +1,38 @@
+pub mod subscript_lexer;
+
+use orchid_extension::tree::{GenMember, comments, prefix};
+
+use crate::proto;
+
+pub fn gen_ops_lib() -> Vec<GenMember> {
+	prefix("std::ops", [
+		comments(
+			["Protocol for the infix + operator", "|type: self -> rhs -> self|"],
+			proto(true, "add").finish(),
+		),
+		comments(
+			["Protocol for the infix - operator", "|type: self -> rhs -> self|"],
+			proto(true, "sub").finish(),
+		),
+		comments(
+			["Protocol for the infix * operator", "|type: self -> rhs -> self|"],
+			proto(true, "mul").finish(),
+		),
+		comments(
+			["Protocol for the infix / operator", "|type: self -> rhs -> self|"],
+			proto(true, "div").finish(),
+		),
+		comments(
+			["Protocol for the infix % operator", "|type: self -> rhs -> self|"],
+			proto(true, "mod").finish(),
+		),
+		comments(
+			["Protocol used by paths for reading", "|type: self -> key -> value|"],
+			proto(true, "get").finish(),
+		),
+		comments(
+			["Protocol used by paths for writing", "|type: self -> key -> value -> self|"],
+			proto(true, "set").finish(),
+		),
+	])
+}

orchid-std/src/std/ops/subscript_lexer.rs

@@ -0,0 +1,29 @@
+use orchid_base::error::OrcRes;
+use orchid_base::interner::is;
+use orchid_base::parse::{name_char, name_start};
+use orchid_extension::gen_expr::new_atom;
+use orchid_extension::lexer::{LexContext, LexedData, Lexer, err_not_applicable};
+use orchid_extension::tree::GenTok;
+
+use crate::std::string::str_atom::IntStrAtom;
+
+#[derive(Default, Debug)]
+pub struct SubscriptLexer;
+impl Lexer for SubscriptLexer {
+	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['.'..='.'];
+	async fn lex<'a>(tail: &'a str, lctx: &'a LexContext<'a>) -> OrcRes<(&'a str, impl LexedData)> {
+		if tail.len() <= 1 || !name_start(tail.chars().nth(1).unwrap()) {
+			return Err(err_not_applicable().await);
+		}
+		let name_len = match tail[1..].char_indices().find(|(_, c)| !name_char(*c)) {
+			None => tail.len() - 1,
+			Some((pos, _)) => pos,
+		};
+		let new_tail = &tail[name_len + 1..];
+		Ok((new_tail, [
+			GenTok::Name(is(".").await).at(lctx.pos_lt(1, &tail[1..])),
+			GenTok::NewExpr(new_atom(IntStrAtom(is(&tail[1..name_len + 1]).await)))
+				.at(lctx.pos_tt(&tail[1..], new_tail)),
+		]))
+	}
+}

orchid-std/src/std/option.rs

@@ -10,7 +10,7 @@ use orchid_extension::atom::{Atomic, ForeignAtom, TAtom};
 use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use orchid_extension::conv::{ToExpr, TryFromExpr};
 use orchid_extension::expr::{Expr, ExprHandle};
-use orchid_extension::gen_expr::{call, sym_ref};
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
 use orchid_extension::tree::{GenMember, cnst, fun, prefix};
 
 use crate::{OrcString, api};
@@ -59,8 +59,8 @@ impl<T: ToExpr + 'static> ToExpr for OrcOpt<T> {
 
 pub fn gen_option_lib() -> Vec<GenMember> {
 	prefix("std::option", [
-		cnst(true, "none", OptAtom(None)),
+		cnst(true, "none", new_atom(OptAtom(None))),
-		fun(true, "some", async |ex: Expr| OptAtom(Some(ex))),
+		fun(true, "some", async |ex: Expr| new_atom(OptAtom(Some(ex)))),
 		fun(true, "expect", async |opt: ForeignAtom, msg: OrcString| {
 			match OrcOpt::try_from_expr(opt.clone().ex()).await? {
 				OrcOpt(Some(ex)) => Ok::<Expr, _>(ex),

orchid-std/src/std/protocol/parse_impls.rs

@@ -18,17 +18,19 @@ pub async fn parse_impls(
 ) -> OrcRes<()> {
 	let body = match &body_tt.tok {
 		Token::S(Paren::Round, body) => line_items(Snippet::new(body_tt, body)).await,
-		Token::S(ptyp, _) =>
+		Token::S(ptyp, _) => {
 			return Err(mk_errv(
 				is("Incorrect paren type").await,
 				format!("Expected () block, found {ptyp}"),
 				[body_tt.sr().pos()],
-			)),
+			));
-		_ =>
+		},
+		_ => {
 			return Err(
 				token_errv(body_tt, "Expected body", |s| format!("Expected (impl ...) block, found {s}"))
 					.await,
-			),
+			);
+		},
 	};
 	for Parsed { tail: line, output: comments } in body {
 		if let Ok(Parsed { tail, .. }) = expect_tok(line, is("impl").await).await {
@@ -37,18 +39,20 @@ pub async fn parse_impls(
 				Ok(None) => panic!("multiname is always at least one name"),
 				Ok(Some(ref n @ Import { name: Some(_), ref sr, .. })) =>
 					(n.clone().mspath().to_sym().await, sr.clone()),
-				Ok(Some(Import { name: None, sr, .. })) =>
+				Ok(Some(Import { name: None, sr, .. })) => {
 					return Err(mk_errv(
 						is("impl line with globstar").await,
 						"::* is not permitted in a protocol impl",
 						[sr.pos()],
-					)),
+					));
-				Err(e) =>
+				},
+				Err(e) => {
 					return Err(mk_errv(
 						is("Impl line with multiple protocol names").await,
 						"::() is not permitted in a protocol impl",
 						e.map(|i| i.sr.pos()),
-					)),
+					));
+				},
 			};
 			let Parsed { tail, .. } = expect_tok(tail, is("as").await).await?;
 			let cnst_name = is(&format!("{}{}", lines.len(), name.iter().join("__"))).await;

orchid-std/src/std/protocol/proto_parser.rs

@@ -7,7 +7,7 @@ use orchid_base::parse::{Comment, Parsed, expect_end, try_pop_no_fluff};
 use orchid_base::sym;
 use orchid_base::tree::Token;
 use orchid_extension::coroutine_exec::exec;
-use orchid_extension::gen_expr::{call, sym_ref};
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
 use orchid_extension::parser::{PSnippet, ParsCtx, ParsedLine, Parser};
 
 use crate::std::protocol::parse_impls::parse_impls;
@@ -44,7 +44,7 @@ impl Parser for AsProtoParser {
 				for (k, v) in impls {
 					new_impls.insert(k.clone(), h.register(sym_ref(id.suffix([v]).await)).await);
 				}
-				Tag { id, impls: Rc::new(new_impls) }
+				new_atom(Tag { id, impls: Rc::new(new_impls) })
 			})
 			.await
 		}));

orchid-std/src/std/protocol/type_parser.rs

@@ -7,7 +7,7 @@ use orchid_base::parse::{Comment, Parsed, expect_end, try_pop_no_fluff};
 use orchid_base::sym;
 use orchid_base::tree::Token;
 use orchid_extension::coroutine_exec::exec;
-use orchid_extension::gen_expr::{call, sym_ref};
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
 use orchid_extension::parser::{PSnippet, ParsCtx, ParsedLine, Parser};
 
 use crate::std::protocol::parse_impls::parse_impls;
@@ -44,7 +44,7 @@ impl Parser for AsTypeParser {
 				for (k, v) in impls {
 					new_impls.insert(k.clone(), h.register(sym_ref(id.suffix([v]).await)).await);
 				}
-				Tag { id, impls: Rc::new(new_impls) }
+				new_atom(Tag { id, impls: Rc::new(new_impls) })
 			})
 			.await
 		}));

orchid-std/src/std/protocol/types.rs

@@ -1,22 +1,31 @@
 use std::borrow::Cow;
+use std::cell::RefCell;
+use std::io;
 use std::rc::Rc;
 
+use futures::FutureExt;
+use futures::future::{LocalBoxFuture, join_all};
 use hashbrown::HashMap;
+use itertools::Itertools;
 use never::Never;
-use orchid_api_derive::Coding;
+use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use orchid_base::error::{OrcRes, mk_errv};
 use orchid_base::format::fmt;
-use orchid_base::interner::is;
+use orchid_base::interner::{ev, is};
-use orchid_base::name::Sym;
+use orchid_base::name::{NameLike, Sym, VName};
+use orchid_base::reqnot::ReqHandleExt;
 use orchid_extension::atom::{AtomMethod, Atomic, ForeignAtom, MethodSetBuilder, Supports, TAtom};
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
-use orchid_extension::conv::ToExpr;
+use orchid_extension::conv::{ClonableToExprDyn, ToExpr};
-use orchid_extension::expr::Expr;
+use orchid_extension::coroutine_exec::exec;
-use orchid_extension::gen_expr::call;
+use orchid_extension::expr::{Expr, ExprHandle};
-use orchid_extension::tree::{GenMember, fun, prefix};
+use orchid_extension::gen_expr::{GExpr, call, new_atom, sym_ref};
+use orchid_extension::system::sys_req;
+use orchid_extension::tree::{GenMember, MemKind, cnst, fun, lazy, prefix};
 
-use crate::api;
+use crate::std::std_system::StdReq;
+use crate::{StdSystem, api};
 
 #[derive(Clone, Debug)]
 pub struct Tag {
@@ -26,32 +35,48 @@ pub struct Tag {
 impl Atomic for Tag {
 	type Data = api::TStrv;
 	type Variant = OwnedVariant;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new().handle::<GetImplMethod>() }
+	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new().handle::<ProtocolMethod>() }
 }
 impl OwnedAtom for Tag {
 	type Refs = Never;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.id.to_api()) }
 }
-impl Supports<GetImplMethod> for Tag {
+impl Supports<ProtocolMethod> for Tag {
-	async fn handle(&self, req: GetImplMethod) -> <GetImplMethod as Request>::Response {
+	async fn handle<'a>(
-		self.impls.get(&Sym::from_api(req.0).await).map(|expr| expr.handle().ticket())
+		&self,
+		hand: Box<dyn orchid_base::reqnot::ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> std::io::Result<orchid_base::reqnot::Receipt<'a>> {
+		match req {
+			ProtocolMethod::GetTagId(req) => hand.reply(&req, &self.id.to_api()).await,
+			ProtocolMethod::GetImpl(ref req @ GetImpl(key)) =>
+				hand
+					.reply(req, &self.impls.get(&Sym::from_api(key).await).map(|expr| expr.handle().ticket()))
+					.await,
+		}
 	}
 }
-#[derive(Clone, Debug, Coding)]
+
-pub struct GetImplMethod(pub api::TStrv);
+#[derive(Clone, Debug, Coding, Hierarchy)]
-impl Request for GetImplMethod {
+#[extends(ProtocolMethod)]
+pub struct GetImpl(pub api::TStrv);
+impl Request for GetImpl {
 	type Response = Option<api::ExprTicket>;
 }
-impl AtomMethod for GetImplMethod {
+#[derive(Clone, Debug, Coding, Hierarchy)]
-	const NAME: &str = "std::protocol::get_impl";
+#[extends(ProtocolMethod)]
-}
+pub struct GetTagId;
-#[derive(Clone, Debug, Coding)]
+impl Request for GetTagId {
-pub struct GetTagIdMethod;
-impl Request for GetTagIdMethod {
 	type Response = api::TStrv;
 }
-impl AtomMethod for GetTagIdMethod {
+#[derive(Clone, Debug, Coding, Hierarchy)]
-	const NAME: &str = "std::protocol::get_tag_id";
+#[extendable]
+pub enum ProtocolMethod {
+	GetTagId(GetTagId),
+	GetImpl(GetImpl),
+}
+impl AtomMethod for ProtocolMethod {
+	const NAME: &str = "std::protocol";
 }
 
 #[derive(Clone, Debug)]
@@ -67,39 +92,45 @@ impl OwnedAtom for Tagged {
 	type Refs = Never;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.tag.id.to_api()) }
 }
-impl Supports<GetImplMethod> for Tagged {
+impl Supports<ProtocolMethod> for Tagged {
-	async fn handle(&self, req: GetImplMethod) -> <GetImplMethod as Request>::Response {
+	async fn handle<'a>(
-		self.tag.handle(req).await
+		&self,
+		hand: Box<dyn orchid_base::reqnot::ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> io::Result<orchid_base::reqnot::Receipt<'a>> {
+		self.tag.handle(hand, req).await
 	}
 }
 
 pub async fn get_impl(receiver: ForeignAtom, proto: ForeignAtom) -> OrcRes<Expr> {
-	let Some(proto_id) = proto.request(GetTagIdMethod).await else {
+	let Some(proto_id) = proto.request(GetTagId).await else {
-		return Err(mk_errv(is("Not a protocol").await, "Protocol does not have a tag ID", [
+		return Err(mk_errv(
-			proto.pos()
+			is("Not a protocol").await,
-		]));
+			format!("Protocol ({}) does not have a tag ID", fmt(&proto).await),
+			[proto.pos()],
+		));
 	};
-	let Some(impl_val_opt) = receiver.request(GetImplMethod(proto_id)).await else {
+	let Some(impl_val_opt) = receiver.request(GetImpl(proto_id)).await else {
 		return Err(mk_errv(
 			is("Receiver not tagged").await,
-			"The receiver does not have a type tag",
+			format!("The receiver ({}) does not have a type tag", fmt(&receiver).await),
 			[receiver.pos()],
 		));
 	};
 	if let Some(impl_val) = impl_val_opt {
 		return Ok(Expr::deserialize(impl_val).await);
 	}
-	let Some(type_id) = receiver.request(GetTagIdMethod).await else {
+	let Some(type_id) = receiver.request(GetTagId).await else {
 		return Err(mk_errv(
 			is("Incorrect protocols implementation in extension").await,
-			"Atom provides an impl table but no tag ID",
+			format!("The receiver ({}) provides an impl table but no tag ID", fmt(&receiver).await),
 			[receiver.pos()],
 		));
 	};
-	let Some(impl_val_opt) = proto.request(GetImplMethod(type_id)).await else {
+	let Some(impl_val_opt) = proto.request(GetImpl(type_id)).await else {
 		return Err(mk_errv(
 			is("Incorrect protocols implementation in extension").await,
-			"Proto table atom provides a tag ID but no impl table",
+			format!("Protocol ({}) provides a tag ID but no impl table", fmt(&proto).await),
 			[receiver.pos()],
 		));
 	};
@@ -108,7 +139,7 @@ pub async fn get_impl(receiver: ForeignAtom, proto: ForeignAtom) -> OrcRes<Expr>
 	}
 	return Err(mk_errv(
 		is("Implementation not found").await,
-		"This protocol is not implemented for this receiver",
+		format!("Protocol {} is not implemented for {}", ev(proto_id).await, ev(type_id).await),
 		[receiver.pos(), proto.pos()],
 	));
 }
@@ -118,7 +149,9 @@ pub fn gen_protocol_lib() -> Vec<GenMember> {
 		fun(false, "resolve", async |tag: ForeignAtom, value: ForeignAtom| {
 			Ok(call(get_impl(value.clone(), tag).await?.to_gen().await, [value.to_gen().await]))
 		}),
-		fun(false, "wrap", async |tag: TAtom<Tag>, value: Expr| Tagged { tag: own(&tag).await, value }),
+		fun(false, "wrap", async |tag: TAtom<Tag>, value: Expr| {
+			new_atom(Tagged { tag: own(&tag).await, value })
+		}),
 		fun(false, "unwrap", async |tag: TAtom<Tag>, value: TAtom<Tagged>| {
 			let own_tag = own(&tag).await;
 			let own_val = own(&value).await;
@@ -134,3 +167,107 @@ pub fn gen_protocol_lib() -> Vec<GenMember> {
 		}),
 	])
 }
+
+#[derive(Debug, Clone, Coding, Hierarchy)]
+#[extends(StdReq)]
+pub struct CreateTag {
+	pub name: api::TStrv,
+	pub impls: std::collections::HashMap<api::TStrv, api::ExprTicket>,
+}
+impl Request for CreateTag {
+	type Response = api::ExprTicket;
+}
+
+pub fn type_tag<'a>(name: &str) -> TagBuilder<'a> {
+	TagBuilder { name: name.to_owned(), impls: HashMap::default() }
+}
+pub struct TagBuilder<'a> {
+	name: String,
+	impls: HashMap<String, LocalBoxFuture<'a, GExpr>>,
+}
+impl<'a> TagBuilder<'a> {
+	pub fn add_impl(&mut self, name: &str, val: impl ToExpr + 'a) {
+		self.impls.insert(name.to_owned(), val.to_gen().boxed_local());
+	}
+	pub fn with_impl(mut self, name: &str, val: impl ToExpr + 'a) -> Self {
+		self.add_impl(name, val);
+		self
+	}
+	pub async fn finish(self) -> TAtom<Tag> {
+		let tk = sys_req::<StdSystem, _>(CreateTag {
+			name: Sym::parse(&self.name).await.unwrap().to_api(),
+			impls: join_all(self.impls.into_iter().map(|(s, fut)| async move {
+				(
+					Sym::parse(&s).await.unwrap().to_api(),
+					fut.await.create().await.handle().serialize().await,
+				)
+			}))
+			.await
+			.into_iter()
+			.collect(),
+		})
+		.await;
+		TAtom::downcast(ExprHandle::deserialize(tk)).await.unwrap()
+	}
+}
+
+pub fn proto(public: bool, name: &str) -> ProtoBuilder {
+	ProtoBuilder { public, name: name.to_owned(), impls: HashMap::new(), body: Vec::new() }
+}
+pub struct ProtoBuilder {
+	public: bool,
+	name: String,
+	impls: HashMap<String, Box<dyn ClonableToExprDyn>>,
+	body: Vec<GenMember>,
+}
+impl ProtoBuilder {
+	pub fn add_impl(&mut self, name: &str, val: impl ToExpr + Clone + 'static) {
+		self.impls.insert(name.to_owned(), Box::new(val));
+	}
+	pub fn with_impl(mut self, name: &str, val: impl ToExpr + Clone + 'static) -> Self {
+		self.add_impl(name, val);
+		self
+	}
+	pub fn add_body(&mut self, members: impl IntoIterator<Item = GenMember>) {
+		self.body.extend(members);
+	}
+	pub fn with_body(mut self, members: impl IntoIterator<Item = GenMember>) -> Self {
+		self.add_body(members);
+		self
+	}
+	pub fn finish(self) -> Vec<GenMember> {
+		lazy(self.public, &self.name, async |path| {
+			let mut tag = type_tag(&path.segs().join("::"));
+			for (name, value) in self.impls {
+				tag.add_impl(&name, value.to_expr().await);
+			}
+			MemKind::module([
+				cnst(true, "__protocol_tag__", tag.finish().await),
+				fun(true, "resolve", resolver_for(path.to_vname())),
+			])
+		})
+	}
+}
+
+pub fn resolver_for(proto: VName) -> impl AsyncFn(ForeignAtom) -> GExpr + Clone {
+	let proto_cache = RefCell::new(None);
+	async move |atom| {
+		let proto_cache = proto_cache.clone();
+		let proto = proto.clone();
+		exec(async move |mut h| {
+			let cached_proto = proto_cache.borrow().as_ref().cloned();
+			let proto = match cached_proto {
+				Some(val) => val,
+				None => {
+					let proto: ForeignAtom = h
+						.exec(sym_ref(proto.clone().suffix([is("__protocol_tag__").await]).to_sym().await))
+						.await?;
+					*proto_cache.borrow_mut() = Some(proto.clone());
+					proto
+				},
+			};
+			Ok(call(get_impl(atom.clone(), proto).await?.to_gen().await, [atom.to_gen().await]))
+		})
+		.await
+	}
+}

orchid-std/src/std/record/record_atom.rs

@@ -1,25 +1,33 @@
 use std::borrow::Cow;
+use std::io;
 use std::pin::Pin;
 use std::rc::Rc;
 
 use futures::AsyncWrite;
 use futures::future::join_all;
 use hashbrown::HashMap;
-use orchid_api_traits::Encode;
+use orchid_api_derive::Coding;
+use orchid_api_traits::{Encode, Request};
 use orchid_base::interner::{IStr, es};
-use orchid_extension::atom::Atomic;
+use orchid_base::name::Sym;
+use orchid_base::reqnot::{Receipt, ReqHandle, ReqHandleExt};
+use orchid_base::sym;
+use orchid_extension::atom::{Atomic, MethodSetBuilder, Supports};
 use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::sym_ref;
 
 use crate::api;
+use crate::std::protocol::types::{GetImpl, ProtocolMethod};
 
 #[derive(Clone)]
-pub struct Record(pub Rc<HashMap<IStr, Expr>>);
+pub struct RecordAtom(pub Rc<HashMap<IStr, Expr>>);
-impl Atomic for Record {
+impl Atomic for RecordAtom {
 	type Data = ();
 	type Variant = OwnedVariant;
+	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new().handle::<ProtocolMethod>() }
 }
-impl OwnedAtom for Record {
+impl OwnedAtom for RecordAtom {
 	type Refs = Vec<Expr>;
 	async fn serialize(&self, write: Pin<&mut (impl AsyncWrite + ?Sized)>) -> Self::Refs {
 		let (keys, values) =
@@ -30,8 +38,36 @@ impl OwnedAtom for Record {
 	async fn deserialize(mut dctx: impl DeserializeCtx, refs: Self::Refs) -> Self {
 		let keys =
 			join_all(dctx.decode::<Vec<api::TStr>>().await.iter().map(|t| async { es(*t).await })).await;
-		Record(Rc::new(keys.into_iter().zip(refs).collect()))
+		RecordAtom(Rc::new(keys.into_iter().zip(refs).collect()))
 	}
 
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
 }
+impl Supports<ProtocolMethod> for RecordAtom {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> io::Result<Receipt<'a>> {
+		match req {
+			ProtocolMethod::GetTagId(req) => hand.reply(&req, &sym!(std::record::Record).to_api()).await,
+			ProtocolMethod::GetImpl(ref req @ GetImpl(key)) => {
+				let name = Sym::from_api(key).await;
+				let val = if name == sym!(std::ops::get) {
+					sym_ref(sym!(std::record::get))
+				} else if name == sym!(std::ops::set) {
+					sym_ref(sym!(std::record::set))
+				} else {
+					return hand.reply(req, &None).await;
+				};
+				return hand.reply(req, &Some(val.create().await.serialize().await)).await;
+			},
+		}
+	}
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct CreateRecord(pub std::collections::HashMap<api::TStr, api::ExprTicket>);
+impl Request for CreateRecord {
+	type Response = api::ExprTicket;
+}

orchid-std/src/std/record/record_lib.rs

@@ -1,30 +1,41 @@
 use std::rc::Rc;
 
 use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::mk_errv;
+use orchid_base::interner::is;
 use orchid_extension::atom::TAtom;
 use orchid_extension::atom_owned::own;
 use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::{arg, new_atom};
 use orchid_extension::tree::{GenMember, cnst, fun, prefix};
 
-use crate::std::option::OrcOpt;
+use crate::std::record::record_atom::RecordAtom;
-use crate::std::record::record_atom::Record;
 use crate::std::string::str_atom::IntStrAtom;
 
 pub fn gen_record_lib() -> Vec<GenMember> {
 	prefix("std::record", [
-		cnst(true, "empty", Record(Rc::new(HashMap::new()))),
+		cnst(true, "empty", new_atom(RecordAtom(Rc::new(HashMap::new())))),
-		fun(true, "set", async |map: TAtom<Record>, key: IntStrAtom, val: Expr| {
+		fun(true, "set", async |map: TAtom<RecordAtom>, key: IntStrAtom, val: Expr| {
 			let mut map = own(&map).await.0.as_ref().clone();
 			map.insert(key.0.clone(), val);
-			Record(Rc::new(map))
+			new_atom(RecordAtom(Rc::new(map)))
 		}),
-		fun(true, "get", async |map: TAtom<Record>, key: IntStrAtom| {
+		fun(true, "get", async |map: TAtom<RecordAtom>, key: IntStrAtom| {
-			OrcOpt(own(&map).await.0.get(&key.0).cloned())
+			let record = own(&map).await;
+			match record.0.get(&key.0) {
+				Some(val) => Ok(val.clone()),
+				None => Err(mk_errv(
+					is("Key not found in record").await,
+					format!("{} is not in this record, valid keys are {}", key.0, record.0.keys().join(", ")),
+					[arg(0).pos.clone(), arg(1).pos.clone()],
+				)),
+			}
 		}),
-		fun(true, "delete", async |map: TAtom<Record>, key: IntStrAtom| {
+		fun(true, "delete", async |map: TAtom<RecordAtom>, key: IntStrAtom| {
 			let mut map = own(&map).await.0.as_ref().clone();
 			map.remove(&key.0);
-			Record(Rc::new(map))
+			new_atom(RecordAtom(Rc::new(map)))
 		}),
 	])
 }

orchid-std/src/std/reflection/sym_atom.rs

@@ -5,10 +5,12 @@ use orchid_api_traits::Request;
 use orchid_base::error::mk_errv;
 use orchid_base::interner::{es, is};
 use orchid_base::name::{NameLike, Sym};
+use orchid_base::reqnot::ReqHandleExt;
 use orchid_extension::atom::{Atomic, Supports, TAtom};
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
 use orchid_extension::expr::{Expr, ExprHandle};
-use orchid_extension::system::dep_req;
+use orchid_extension::gen_expr::new_atom;
+use orchid_extension::system::sys_req;
 use orchid_extension::tree::{GenMember, fun, prefix};
 
 use crate::std::std_system::StdReq;
@@ -29,8 +31,12 @@ impl OwnedAtom for SymAtom {
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(SymAtomData(self.0.tok().to_api())) }
 }
 impl Supports<ToStringMethod> for SymAtom {
-	async fn handle(&self, _: ToStringMethod) -> <ToStringMethod as Request>::Response {
+	async fn handle<'a>(
-		self.0.to_string()
+		&self,
+		hand: Box<dyn orchid_base::reqnot::ReqHandle<'a> + '_>,
+		req: ToStringMethod,
+	) -> std::io::Result<orchid_base::reqnot::Receipt<'a>> {
+		hand.reply(&req, &self.0.to_string()).await
 	}
 }
 
@@ -43,7 +49,7 @@ impl Request for CreateSymAtom {
 
 pub async fn sym_expr(sym: Sym) -> Expr {
 	Expr::from_handle(ExprHandle::deserialize(
-		dep_req::<StdSystem, _>(CreateSymAtom(sym.to_api())).await,
+		sys_req::<StdSystem, _>(CreateSymAtom(sym.to_api())).await,
 	))
 }
 
@@ -51,7 +57,7 @@ pub async fn gen_sym_lib() -> Vec<GenMember> {
 	prefix("std::refl::sym", [
 		fun(true, "from_str", async move |str: TAtom<IntStrAtom>| {
 			match Sym::parse(&es(*str).await).await {
-				Ok(sym) => Ok(SymAtom(sym)),
+				Ok(sym) => Ok(new_atom(SymAtom(sym))),
 				Err(_) => Err(mk_errv(
 					is("Cannot parse sym from empty string").await,
 					"Empty string passed to std::refl::sym::from_str",
@@ -60,7 +66,7 @@ pub async fn gen_sym_lib() -> Vec<GenMember> {
 			}
 		}),
 		fun(true, "to_tpl", async move |sym: TAtom<SymAtom>| {
-			HomoTpl(own(&sym).await.0.segs().map(IntStrAtom).collect())
+			HomoTpl(own(&sym).await.0.segs().map(|seg| new_atom(IntStrAtom(seg))).collect())
 		}),
 	])
 }

orchid-std/src/std/std_system.rs

@@ -2,12 +2,14 @@ use std::rc::Rc;
 
 use futures::future::join_all;
 use orchid_api_derive::{Coding, Hierarchy};
+use orchid_base::interner::es;
 use orchid_base::name::Sym;
 use orchid_base::reqnot::{Receipt, ReqHandle, ReqHandleExt};
 use orchid_base::sym;
 use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
 use orchid_extension::conv::ToExpr;
 use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::new_atom;
 use orchid_extension::lexer::LexerObj;
 use orchid_extension::parser::ParserObj;
 use orchid_extension::system::{System, SystemCard};
@@ -17,23 +19,33 @@ use orchid_extension::tree::{GenMember, merge_trivial};
 use super::number::num_lib::gen_num_lib;
 use super::string::str_atom::{IntStrAtom, StrAtom};
 use super::string::str_lib::gen_str_lib;
+use crate::std::binary::binary_atom::BlobAtom;
+use crate::std::binary::binary_lib::gen_binary_lib;
+use crate::std::boolean::gen_bool_lib;
+use crate::std::number::num_atom::{CreateFloat, CreateInt};
 use crate::std::number::num_lexer::NumLexer;
+use crate::std::ops::gen_ops_lib;
+use crate::std::ops::subscript_lexer::SubscriptLexer;
 use crate::std::option::{OptAtom, gen_option_lib};
 use crate::std::protocol::proto_parser::{AsProtoParser, ProtoParser};
 use crate::std::protocol::type_parser::{AsTypeParser, TypeParser};
-use crate::std::protocol::types::{Tag, Tagged, gen_protocol_lib};
+use crate::std::protocol::types::{CreateTag, Tag, Tagged, gen_protocol_lib};
-use crate::std::record::record_atom::Record;
+use crate::std::record::record_atom::{CreateRecord, RecordAtom};
 use crate::std::record::record_lib::gen_record_lib;
 use crate::std::reflection::sym_atom::{CreateSymAtom, SymAtom, gen_sym_lib};
 use crate::std::string::str_lexer::StringLexer;
-use crate::std::string::to_string::AsStrTag;
 use crate::std::tuple::{CreateTuple, Tuple, TupleBuilder, gen_tuple_lib};
 use crate::{Float, Int};
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extendable]
+#[allow(clippy::enum_variant_names, reason = "For the time being there are only ctor calls")]
 pub enum StdReq {
+	CreateInt(CreateInt),
+	CreateFloat(CreateFloat),
+	CreateTag(CreateTag),
 	CreateTuple(CreateTuple),
+	CreateRecord(CreateRecord),
 	CreateSymAtom(CreateSymAtom),
 }
 
@@ -51,38 +63,66 @@ impl SystemCard for StdSystem {
 	type Req = StdReq;
 	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
 		[
+			Some(BlobAtom::dynfo()),
 			Some(Int::dynfo()),
 			Some(Float::dynfo()),
 			Some(StrAtom::dynfo()),
 			Some(IntStrAtom::dynfo()),
 			Some(OptAtom::dynfo()),
-			Some(Record::dynfo()),
+			Some(RecordAtom::dynfo()),
 			Some(Tuple::dynfo()),
 			Some(TupleBuilder::dynfo()),
 			Some(Tag::dynfo()),
 			Some(Tagged::dynfo()),
-			Some(AsStrTag::dynfo()),
 		]
 	}
 }
 impl System for StdSystem {
 	async fn request<'a>(xreq: Box<dyn ReqHandle<'a> + 'a>, req: Self::Req) -> Receipt<'a> {
 		match req {
+			StdReq::CreateInt(ref req @ CreateInt(int)) =>
+				xreq.reply(req, &new_atom(int).to_expr().await.serialize().await).await.unwrap(),
+			StdReq::CreateFloat(ref req @ CreateFloat(float)) =>
+				xreq.reply(req, &new_atom(float).to_expr().await.serialize().await).await.unwrap(),
 			StdReq::CreateTuple(ref req @ CreateTuple(ref items)) => {
 				let tpl = Tuple(Rc::new(join_all(items.iter().copied().map(Expr::deserialize)).await));
-				let tk = tpl.to_expr().await.serialize().await;
+				let tk = new_atom(tpl).to_expr().await.serialize().await;
+				xreq.reply(req, &tk).await.unwrap()
+			},
+			StdReq::CreateRecord(ref req @ CreateRecord(ref items)) => {
+				let values =
+					join_all(items.iter().map(async |(k, v)| (es(*k).await, Expr::deserialize(*v).await)))
+						.await;
+				let rec = RecordAtom(Rc::new(values.into_iter().collect()));
+				let tk = new_atom(rec).to_expr().await.serialize().await;
 				xreq.reply(req, &tk).await.unwrap()
 			},
 			StdReq::CreateSymAtom(ref req @ CreateSymAtom(sym_tok)) => {
 				let sym_atom = SymAtom(Sym::from_api(sym_tok).await);
-				xreq.reply(req, &sym_atom.to_expr().await.serialize().await).await.unwrap()
+				xreq.reply(req, &new_atom(sym_atom).to_expr().await.serialize().await).await.unwrap()
+			},
+			StdReq::CreateTag(ref req @ CreateTag { name, ref impls }) => {
+				let tag_atom = Tag {
+					id: Sym::from_api(name).await,
+					impls: Rc::new(
+						join_all(
+							(impls.iter())
+								.map(|(k, v)| async { (Sym::from_api(*k).await, Expr::deserialize(*v).await) }),
+						)
+						.await
+						.into_iter()
+						.collect(),
+					),
+				};
+				xreq.reply(req, &new_atom(tag_atom).to_expr().await.serialize().await).await.unwrap()
 			},
 		}
 	}
-	fn lexers() -> Vec<LexerObj> { vec![&StringLexer, &NumLexer] }
+	fn lexers() -> Vec<LexerObj> { vec![&StringLexer, &NumLexer, &SubscriptLexer] }
 	fn parsers() -> Vec<ParserObj> { vec![&AsTypeParser, &TypeParser, &AsProtoParser, &ProtoParser] }
 	async fn env() -> Vec<GenMember> {
 		merge_trivial([
+			gen_bool_lib(),
 			gen_num_lib(),
 			gen_str_lib(),
 			gen_option_lib(),
@@ -90,7 +130,11 @@ impl System for StdSystem {
 			gen_tuple_lib(),
 			gen_protocol_lib(),
 			gen_sym_lib().await,
+			gen_ops_lib(),
+			gen_binary_lib(),
 		])
 	}
-	async fn prelude() -> Vec<Sym> { vec![sym!(std), sym!(std::tuple), sym!(std::option)] }
+	async fn prelude() -> Vec<Sym> {
+		vec![sym!(std), sym!(std::tuple), sym!(std::option), sym!(std::record), sym!(std::string)]
+	}
 }

orchid-std/src/std/string/str_atom.rs

@@ -1,44 +1,47 @@
 use std::borrow::Cow;
+use std::io;
 use std::ops::Deref;
 use std::pin::Pin;
 use std::rc::Rc;
 
 use futures::AsyncWrite;
-use orchid_api_derive::Coding;
+use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{Encode, Request};
 use orchid_base::error::{OrcRes, mk_errv};
 use orchid_base::format::{FmtCtx, FmtUnit};
 use orchid_base::interner::{IStr, es, is};
+use orchid_base::name::Sym;
+use orchid_base::reqnot::{Receipt, ReqHandle, ReqHandleExt};
+use orchid_base::sym;
 use orchid_extension::atom::{AtomMethod, Atomic, MethodSetBuilder, Supports, TAtom};
 use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use orchid_extension::conv::TryFromExpr;
 use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::sym_ref;
 
+use crate::std::protocol::types::{GetImpl, ProtocolMethod};
 use crate::std::string::to_string::ToStringMethod;
 
-#[derive(Copy, Clone, Debug, Coding)]
+#[derive(Copy, Clone, Debug, Coding, Hierarchy)]
-pub struct StringGetVal;
+pub struct StringGetValMethod;
-impl Request for StringGetVal {
+impl Request for StringGetValMethod {
 	type Response = Rc<String>;
 }
-impl AtomMethod for StringGetVal {
+impl AtomMethod for StringGetValMethod {
 	const NAME: &str = "std::string_get_val";
 }
-impl Supports<StringGetVal> for StrAtom {
-	async fn handle(&self, _: StringGetVal) -> <StringGetVal as Request>::Response { self.0.clone() }
-}
-impl Supports<ToStringMethod> for StrAtom {
-	async fn handle(&self, _: ToStringMethod) -> <ToStringMethod as Request>::Response {
-		self.0.as_str().to_string()
-	}
-}
 
 #[derive(Clone)]
 pub struct StrAtom(Rc<String>);
 impl Atomic for StrAtom {
 	type Variant = OwnedVariant;
 	type Data = ();
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new().handle::<StringGetVal>() }
+	fn reg_reqs() -> MethodSetBuilder<Self> {
+		MethodSetBuilder::new()
+			.handle::<StringGetValMethod>()
+			.handle::<ToStringMethod>()
+			.handle::<ProtocolMethod>()
+	}
 }
 impl StrAtom {
 	pub fn new(str: Rc<String>) -> Self { Self(str) }
@@ -60,12 +63,53 @@ impl OwnedAtom for StrAtom {
 		Self::new(Rc::new(ctx.read::<String>().await))
 	}
 }
+impl Supports<StringGetValMethod> for StrAtom {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: StringGetValMethod,
+	) -> io::Result<Receipt<'a>> {
+		hand.reply(&req, &self.0).await
+	}
+}
+impl Supports<ToStringMethod> for StrAtom {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ToStringMethod,
+	) -> io::Result<Receipt<'a>> {
+		hand.reply(&req, &self.0).await
+	}
+}
+impl Supports<ProtocolMethod> for StrAtom {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> io::Result<Receipt<'a>> {
+		match req {
+			ProtocolMethod::GetTagId(req) => hand.reply(&req, &sym!(std::string::StrAtom).to_api()).await,
+			ProtocolMethod::GetImpl(ref req @ GetImpl(key)) => {
+				let name = Sym::from_api(key).await;
+				let val = if name == sym!(std::ops::add) {
+					sym_ref(sym!(std::string::concat))
+				} else {
+					return hand.reply(req, &None).await;
+				};
+				hand.reply(req, &Some(val.create().await.serialize().await)).await
+			},
+		}
+	}
+}
 
 #[derive(Debug, Clone)]
 pub struct IntStrAtom(pub(crate) IStr);
 impl Atomic for IntStrAtom {
 	type Variant = OwnedVariant;
 	type Data = orchid_api::TStr;
+	fn reg_reqs() -> MethodSetBuilder<Self> {
+		MethodSetBuilder::new().handle::<ProtocolMethod>().handle::<ToStringMethod>()
+	}
 }
 impl From<IStr> for IntStrAtom {
 	fn from(value: IStr) -> Self { Self(value) }
@@ -90,11 +134,35 @@ impl TryFromExpr for IntStrAtom {
 	}
 }
 impl Supports<ToStringMethod> for IntStrAtom {
-	async fn handle(&self, _: ToStringMethod) -> <ToStringMethod as Request>::Response {
+	async fn handle<'a>(
-		self.0.to_string()
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ToStringMethod,
+	) -> io::Result<Receipt<'a>> {
+		hand.reply(&req, &self.0.rc()).await
+	}
+}
+impl Supports<ProtocolMethod> for IntStrAtom {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> io::Result<Receipt<'a>> {
+		match req {
+			ProtocolMethod::GetTagId(req) =>
+				hand.reply(&req, &sym!(std::string::IntStrAtom).to_api()).await,
+			ProtocolMethod::GetImpl(ref req @ GetImpl(key)) => {
+				let name = Sym::from_api(key).await;
+				let val = if name == sym!(std::ops::add) {
+					sym_ref(sym!(std::string::concat))
+				} else {
+					return hand.reply(req, &None).await;
+				};
+				hand.reply(req, &Some(val.create().await.serialize().await)).await
+			},
+		}
 	}
 }
-
 #[derive(Clone)]
 pub struct OrcString {
 	kind: OrcStringKind,
@@ -109,7 +177,7 @@ impl OrcString {
 	pub async fn get_string(&self) -> Rc<String> {
 		match &self.kind {
 			OrcStringKind::Int(tok) => es(**tok).await.rc(),
-			OrcStringKind::Val(atom) => atom.request(StringGetVal).await,
+			OrcStringKind::Val(atom) => atom.request(StringGetValMethod).await,
 		}
 	}
 }

orchid-std/src/std/string/str_lexer.rs

@@ -5,7 +5,7 @@ use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::sym;
 use orchid_base::tree::{Paren, wrap_tokv};
-use orchid_extension::gen_expr::sym_ref;
+use orchid_extension::gen_expr::{new_atom, sym_ref};
 use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
 use orchid_extension::parser::p_tree2gen;
 use orchid_extension::tree::{GenTok, GenTokTree, ref_tok, x_tok};
@@ -99,6 +99,7 @@ impl Lexer for StringLexer {
 	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"', '`'..='`'];
 	async fn lex<'a>(all: &'a str, lctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
 		let Some(mut tail) = all.strip_prefix('"') else {
+			eprintln!("Unrecognized start char {:?}", all.chars().next().unwrap());
 			return Err(err_not_applicable().await);
 		};
 		let mut ret = None;
@@ -115,8 +116,8 @@ impl Lexer for StringLexer {
 				err.extend(e.clone().into_proj(ctx.src(), ctx.pos(tail) - str.len() as u32).await);
 			}
 			let str_val = str_val_res.unwrap_or_default();
-			x_tok(IntStrAtom::from(is(&str_val).await)).await.at(ctx.pos_lt(str.len() as u32, tail))
+			let atom = new_atom(IntStrAtom::from(is(&str_val).await));
-				as GenTokTree
+			x_tok(atom).await.at(ctx.pos_lt(str.len() as u32, tail)) as GenTokTree
 		}
 		let add_frag = |prev: Option<GenTokTree>, new: GenTokTree| async {
 			let Some(prev) = prev else { return new };

orchid-std/src/std/string/str_lib.rs

@@ -1,59 +1,174 @@
 use std::rc::Rc;
 
+use orchid_base::error::mk_errv;
 use orchid_base::format::fmt;
+use orchid_base::interner::is;
 use orchid_base::sym;
 use orchid_extension::atom::ForeignAtom;
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::exec;
 use orchid_extension::expr::Expr;
-use orchid_extension::gen_expr::{call, sym_ref};
+use orchid_extension::func_atom::get_arg;
-use orchid_extension::tree::{GenMember, cnst, comments, fun, prefix};
+use orchid_extension::gen_expr::{call, new_atom, sym_ref};
+use orchid_extension::tree::{GenMember, comments, fun, prefix};
+use unicode_segmentation::UnicodeSegmentation;
 
 use super::str_atom::StrAtom;
-use crate::OrcString;
+use crate::std::protocol::types::{get_impl, proto};
-use crate::std::protocol::types::get_impl;
+use crate::std::string::to_string::ToStringMethod;
-use crate::std::string::to_string::{AsStrTag, ToStringMethod};
+use crate::{Int, OrcOpt, OrcString, Tpl};
 
 pub fn gen_str_lib() -> Vec<GenMember> {
-	prefix("std::string", [
+	prefix("std", [comments(
-		comments(
+		["There are two string types, IntStr and Str. Literals are always IntStr, which are quick to \
-			["Concatenate two strings"],
+			equality-compare but may leak, so you can't generally create them at runtime.\n\n\
-			fun(true, "concat", async |left: OrcString, right: OrcString| {
+			All functions here operate on Unicode graphemes. This essentially means that letters with \
-				StrAtom::new(Rc::new(left.get_string().await.to_string() + &right.get_string().await))
+			added diacritics and Mandarin multi-codepoint characters are treated as a single character."],
-			}),
+		prefix("string", [
-		),
+			comments(
-		comments(
+				["Concatenate two strings", "|type: Str -> Str -> Str|"],
-			["Converts a value to string. This function is used in interpolation. \
+				fun(true, "concat", async |left: OrcString, right: OrcString| {
-			It supports the std::string::to_string protocol in Orchid, \
+					new_atom(StrAtom::new(Rc::new(
-			the std::string::to_string request in Rust, \
+						left.get_string().await.to_string() + &right.get_string().await,
-			and expression debug printing as a fallback (print_atom for Atomic implementors in Rust).\n\n\
+					)))
-			This function is infallible."],
+				}),
-			fun(true, "to_str", async |input: Expr| {
+			),
-				exec(async move |mut h| {
+			comments(
-					if let Ok(atom) = h.exec::<ForeignAtom>(input.clone()).await {
+				[
-						if let Some(str) = atom.request(ToStringMethod).await {
+					"Find the size of a string in bytes. Strings are stored in UTF-8. \
-							return StrAtom::new(Rc::new(str)).to_gen().await;
+					This should be used to determine the computational resource utilization of strings. \
-						}
+					It should not be used to determine whether to truncate text.",
-						let proto_ref = sym_ref(sym!(std::string::to_string::__protocol_tag__));
+					"|type: Str -> Int|",
-						let proto = h.exec(proto_ref).await.expect("This protocol is defined in this system");
+				],
-						if let Ok(cb) = get_impl(atom.clone(), proto).await {
+				fun(true, "size", async |s: OrcString| Int(s.get_string().await.len().try_into().unwrap())),
-							return call(cb.to_gen().await, [atom.to_gen().await]).to_gen().await;
+			),
-						}
+			comments(
+				[
+					"Find the number of characters in a string. This can be used for example to \
+					truncate text. It should not be used to limit the size of messages for security purposes.",
+					"|type: Str -> Int|",
+				],
+				fun(true, "len", async |s: OrcString| {
+					Int(s.get_string().await.graphemes(true).count().try_into().unwrap())
+				}),
+			),
+			comments(
+				[
+					"Takes a string, a start and a length in graphemes. \
+					Slices out the specified subsection of the string.",
+					"|type: Str -> Int -> Int -> Str|",
+				],
+				fun(true, "slice", async |s: OrcString, Int(start): Int, Int(len): Int| {
+					let str = s.get_string().await;
+					if len <= 0 {
+						return Ok(new_atom(StrAtom::new(Rc::default())));
 					}
-					return StrAtom::new(Rc::new(fmt(&input).await)).to_gen().await;
+					let mut substr_iter = str.graphemes(true).skip(start.try_into().unwrap());
-				})
+					let new_str: String =
-				.await
+						substr_iter.by_ref().take(usize::try_from(len).unwrap() - 1).collect();
-			}),
+					let Some(s) = substr_iter.next() else {
-		),
+						let str_len = str.graphemes(true).count();
-		prefix("to_string", [
+						return Err(mk_errv(
-			cnst(true, "__type_tag__", AsStrTag),
+							is("Index out of bounds").await,
-			fun(true, "resolve", async |atom: ForeignAtom| {
+							format!("Tried to select grapheme {start}+{len} from string that only has {str_len}"),
-				exec(async |mut h| {
+							[get_arg(0).pos().await, get_arg(1).pos().await, get_arg(2).pos().await],
-					let proto = h.exec(sym_ref(sym!(std::string::to_string))).await?;
+						));
-					Ok(call(get_impl(atom.clone(), proto).await?.to_gen().await, [atom.to_gen().await]))
+					};
-				})
+					Ok(new_atom(StrAtom::new(Rc::new(new_str + s))))
-				.await
+				}),
-			}),
+			),
+			comments(
+				[
+					"If the first string contains the second then returns the index.",
+					"|type: Str -> Str -> std::option Int|",
+				],
+				fun(true, "find", async |haystack: OrcString, needle: OrcString| {
+					let haystack_str = haystack.get_string().await;
+					let needle_str = needle.get_string().await;
+					let mut haystack_graphs = haystack_str.graphemes(true);
+					let mut index = 0;
+					loop {
+						let mut needle_graphs = needle_str.graphemes(true);
+						// check that all chars are equal
+						if haystack_graphs.clone().zip(needle_graphs.by_ref()).all(|(l, r)| l == r) {
+							// if we exhausted the haystack but not the needle, we can't succeed
+							if needle_graphs.next().is_some() {
+								break;
+							}
+							return OrcOpt(Some(Int(index)));
+						}
+						if haystack_graphs.next().is_none() {
+							break;
+						}
+						index += 1;
+					}
+					OrcOpt(None)
+				}),
+			),
+			comments(
+				[
+					"Splits the string into two substrings at the nth grapheme.",
+					"|type: Str -> Int -> std::tuple Str Str|",
+				],
+				fun(true, "split", async |s: OrcString, i: Int| {
+					let str = s.get_string().await;
+					let Some((i, _)) = str.grapheme_indices(true).nth(i.0.try_into().unwrap()) else {
+						let len = str.graphemes(true).count();
+						return Err(mk_errv(
+							is("Index out of bounds").await,
+							format!("Tried to split string at {}, it only has {} graphemes", i.0, len),
+							[get_arg(0).pos().await, get_arg(1).pos().await],
+						));
+					};
+					let (left, right) = str.split_at(i);
+					Ok(Tpl((
+						new_atom(StrAtom::new(Rc::new(left.to_string()))),
+						new_atom(StrAtom::new(Rc::new(right.to_string()))),
+					)))
+				}),
+			),
+			comments(
+				["Returns the nth grapheme.", "|type: Str -> Int -> Str|"],
+				fun(true, "char_at", async |s: OrcString, i: Int| {
+					let str = s.get_string().await;
+					let Some(s) = str.graphemes(true).nth(i.0.try_into().unwrap()) else {
+						let len = str.graphemes(true).count();
+						return Err(mk_errv(
+							is("Index out of bounds").await,
+							format!("Tried to read grapheme {} from string, it only has {}", i.0, len),
+							[get_arg(0).pos().await, get_arg(1).pos().await],
+						));
+					};
+					Ok(new_atom(StrAtom::new(Rc::new(s.to_string()))))
+				}),
+			),
+			comments(
+				[
+					"Converts a value to string. This function is used in interpolation. \
+					It supports the std::string::to_string protocol in Orchid, \
+					the std::string::to_string request in Rust, \
+					and expression debug printing as a fallback (print_atom for Atomic implementors in Rust).\n\n\
+					This function is infallible.",
+					"|type: any -> Str|",
+				],
+				fun(true, "to_str", async |input: Expr| {
+					exec(async move |mut h| {
+						if let Ok(atom) = h.exec::<ForeignAtom>(input.clone()).await {
+							if let Some(str) = atom.request(ToStringMethod).await {
+								return new_atom(StrAtom::new(Rc::new(str)));
+							}
+							let proto_ref = sym_ref(sym!(std::string::to_string::__protocol_tag__));
+							let proto = h.exec(proto_ref).await.expect("This protocol is defined in this system");
+							if let Ok(cb) = get_impl(atom.clone(), proto).await {
+								return call(cb.to_gen().await, [atom.to_gen().await]).to_gen().await;
+							}
+						}
+						return new_atom(StrAtom::new(Rc::new(fmt(&input).await)));
+					})
+					.await
+				}),
+			),
+			proto(true, "to_string").finish(),
 		]),
-	])
+	)])
 }

orchid-std/src/std/string/to_string.rs

@@ -1,31 +1,9 @@
-use orchid_api_derive::Coding;
+use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
-use orchid_base::name::Sym;
+use orchid_extension::atom::AtomMethod;
-use orchid_extension::atom::{AtomMethod, Atomic, MethodSetBuilder, Supports};
-use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
 
-use crate::std::protocol::types::{GetImplMethod, GetTagIdMethod};
+/// Method version of std::string::to_string protocol for atoms
-
+#[derive(Coding, Clone, Debug, Hierarchy)]
-#[derive(Coding, Clone, Debug)]
-pub struct AsStrTag;
-impl Atomic for AsStrTag {
-	type Data = AsStrTag;
-	type Variant = ThinVariant;
-	fn reg_reqs() -> MethodSetBuilder<Self> {
-		MethodSetBuilder::new().handle::<GetTagIdMethod>().handle::<GetImplMethod>()
-	}
-}
-impl ThinAtom for AsStrTag {}
-impl Supports<GetTagIdMethod> for AsStrTag {
-	async fn handle(&self, _: GetTagIdMethod) -> <GetTagIdMethod as Request>::Response {
-		Sym::parse("std::string::to_string").await.unwrap().to_api()
-	}
-}
-impl Supports<GetImplMethod> for AsStrTag {
-	async fn handle(&self, _: GetImplMethod) -> <GetImplMethod as Request>::Response { None }
-}
-
-#[derive(Coding, Clone, Debug)]
 pub struct ToStringMethod;
 impl Request for ToStringMethod {
 	type Response = String;

orchid-std/src/std/tuple.rs

@@ -11,14 +11,18 @@ use orchid_api_traits::Request;
 use orchid_base::error::{OrcRes, mk_errv};
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::interner::is;
-use orchid_extension::atom::{Atomic, TAtom};
+use orchid_base::name::Sym;
+use orchid_base::reqnot::ReqHandleExt;
+use orchid_base::sym;
+use orchid_extension::atom::{Atomic, MethodSetBuilder, Supports, TAtom};
 use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant, own};
 use orchid_extension::conv::{ToExpr, TryFromExpr};
 use orchid_extension::expr::{Expr, ExprHandle};
-use orchid_extension::gen_expr::GExpr;
+use orchid_extension::gen_expr::{GExpr, new_atom, sym_ref};
-use orchid_extension::system::dep_req;
+use orchid_extension::system::sys_req;
 use orchid_extension::tree::{GenMember, cnst, fun, prefix};
 
+use crate::std::protocol::types::{GetImpl, ProtocolMethod};
 use crate::std::std_system::StdReq;
 use crate::{Int, StdSystem, api};
 
@@ -28,6 +32,9 @@ pub struct Tuple(pub(super) Rc<Vec<Expr>>);
 impl Atomic for Tuple {
 	type Data = Vec<api::ExprTicket>;
 	type Variant = OwnedVariant;
+	fn reg_reqs() -> orchid_extension::atom::MethodSetBuilder<Self> {
+		MethodSetBuilder::new().handle::<ProtocolMethod>()
+	}
 }
 
 impl OwnedAtom for Tuple {
@@ -46,6 +53,28 @@ impl OwnedAtom for Tuple {
 			.units_own(join_all(self.0.iter().map(|x| x.print(c))).await)
 	}
 }
+impl Supports<ProtocolMethod> for Tuple {
+	async fn handle<'a>(
+		&self,
+		hand: Box<dyn orchid_base::reqnot::ReqHandle<'a> + '_>,
+		req: ProtocolMethod,
+	) -> std::io::Result<orchid_base::reqnot::Receipt<'a>> {
+		match req {
+			ProtocolMethod::GetTagId(req) => hand.reply(&req, &sym!(std::tuple).to_api()).await,
+			ProtocolMethod::GetImpl(ref req @ GetImpl(key)) => {
+				let name = Sym::from_api(key).await;
+				let val = if name == sym!(std::ops::get) {
+					sym_ref(sym!(std::tuple::get))
+				} else if name == sym!(std::ops::set) {
+					sym_ref(sym!(std::tuple::set))
+				} else {
+					return hand.reply(req, &None).await;
+				};
+				hand.reply(req, &Some(val.create().await.serialize().await)).await
+			},
+		}
+	}
+}
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(StdReq)]
@@ -69,22 +98,22 @@ impl OwnedAtom for TupleBuilder {
 	async fn call(mut self, arg: Expr) -> GExpr {
 		self.items.push(arg);
 		if self.arity.get() == self.items.len().try_into().expect("counting up from 0") {
-			Tuple(Rc::new(self.items)).to_gen().await
+			new_atom(Tuple(Rc::new(self.items)))
 		} else {
-			self.to_gen().await
+			new_atom(self)
 		}
 	}
 }
 
 pub fn gen_tuple_lib() -> Vec<GenMember> {
 	prefix("std::tuple", [
-		cnst(true, "empty", Tuple(Rc::new(Vec::new()))),
+		cnst(true, "empty", new_atom(Tuple(Rc::new(Vec::new())))),
-		fun(true, "one", async |item: Expr| Tuple(Rc::new(vec![item]))),
+		fun(true, "one", async |item: Expr| new_atom(Tuple(Rc::new(vec![item])))),
 		fun(true, "new", async |arity: TAtom<Int>| {
 			if let Ok(arity) = u32::try_from(arity.value.0).and_then(|v| v.try_into()) {
-				TupleBuilder { arity, items: Vec::new() }.to_gen().await
+				new_atom(TupleBuilder { arity, items: Vec::new() })
 			} else {
-				Tuple(Rc::new(Vec::new())).to_gen().await
+				new_atom(Tuple(Rc::new(Vec::new())))
 			}
 		}),
 		fun(true, "get", async |tup: TAtom<Tuple>, idx: TAtom<Int>| {
@@ -104,7 +133,7 @@ pub fn gen_tuple_lib() -> Vec<GenMember> {
 				let mut new_vec = own(&tup).await.0.to_vec();
 				if let Some(slot) = new_vec.get_mut(idx) {
 					*slot = val;
-					return Ok(Tuple(Rc::new(new_vec)));
+					return Ok(new_atom(Tuple(Rc::new(new_vec))));
 				}
 			}
 			return Err(mk_errv(
@@ -117,7 +146,9 @@ pub fn gen_tuple_lib() -> Vec<GenMember> {
 			Int(tup.len().try_into().expect("Tuple was created with an Int length"))
 		}),
 		fun(true, "cat", async |left: TAtom<Tuple>, right: TAtom<Tuple>| {
-			Tuple(Rc::new(own(&left).await.0.iter().chain(own(&right).await.0.iter()).cloned().collect()))
+			new_atom(Tuple(Rc::new(
+				own(&left).await.0.iter().chain(own(&right).await.0.iter()).cloned().collect(),
+			)))
 		}),
 	])
 }
@@ -135,7 +166,7 @@ impl TryFromExpr for UntypedTuple {
 impl ToExpr for UntypedTuple {
 	async fn to_gen(self) -> GExpr {
 		let exprs = join_all(self.0.into_iter().map(async |expr| expr.serialize().await)).await;
-		Expr::deserialize(dep_req::<StdSystem, _>(CreateTuple(exprs)).await).await.to_gen().await
+		Expr::deserialize(sys_req::<StdSystem, _>(CreateTuple(exprs)).await).await.to_gen().await
 	}
 }
 

orcx/src/main.rs

@@ -4,10 +4,13 @@ use tokio::time::Instant;
 pub mod parse_folder;
 
 use std::cell::RefCell;
+use std::collections::HashMap;
 use std::fs::File;
 use std::io::{Read, Write};
+use std::pin::pin;
 use std::process::{Command, ExitCode};
 use std::rc::Rc;
+use std::time::Duration;
 
 use async_fn_stream::try_stream;
 use camino::{Utf8Path, Utf8PathBuf};
@@ -61,7 +64,7 @@ pub struct Args {
 #[derive(Subcommand, Debug)]
 pub enum Commands {
 	Lex {
-		#[arg(short, long)]
+		#[arg()]
 		file: Utf8PathBuf,
 	},
 	Parse {
@@ -179,7 +182,7 @@ impl Spawner for SpawnerImpl {
 
 #[tokio::main]
 async fn main() -> io::Result<ExitCode> {
-	eprintln!("Orcx was made by Lawrence Bethlenfalvy");
+	eprintln!("Orcx v0.1 is free software provided without warranty.");
 	let args = Args::parse();
 	let exit_code = Rc::new(RefCell::new(ExitCode::SUCCESS));
 	let local_set = LocalSet::new();
@@ -187,252 +190,301 @@ async fn main() -> io::Result<ExitCode> {
 	let logger = get_logger(&args);
 	let logger2 = logger.clone();
 	unsafe { STARTUP = Some(Instant::now()) };
-	local_set.spawn_local(with_stash(async move {
+	let ctx = Ctx::new(SpawnerImpl, logger2);
-		let ctx = &Ctx::new(SpawnerImpl, logger2);
+	let (signal_end_main, on_end_main) = futures::channel::oneshot::channel();
-		let extensions = get_all_extensions(&args, ctx).try_collect::<Vec<Extension>>().await.unwrap();
+	let ctx1 = ctx.clone();
-		time_print(&args, "Extensions loaded");
+	local_set.spawn_local(async move {
-		match args.command {
+		let ctx = &ctx1;
-			Commands::Lex { file } => {
+		with_stash(async move {
-				let (_, systems) = init_systems(&args.system, &extensions).await.unwrap();
+			let extensions =
-				let mut file = File::open(file.as_std_path()).unwrap();
+				get_all_extensions(&args, ctx).try_collect::<Vec<Extension>>().await.unwrap();
-				let mut buf = String::new();
+			time_print(&args, "Extensions loaded");
-				file.read_to_string(&mut buf).unwrap();
+			match args.command {
-				let lexemes = lex(is(&buf).await, sym!(usercode), &systems, ctx).await.unwrap();
+				Commands::Lex { file } => {
-				println!("{}", take_first(&ttv_fmt(&lexemes, &FmtCtxImpl::default()).await, true))
+					let (_, systems) = init_systems(&args.system, &extensions).await.unwrap();
-			},
+					let mut file = File::open(file.as_std_path()).unwrap();
-			Commands::Parse { file } => {
+					let mut buf = String::new();
-				let (_, systems) = init_systems(&args.system, &extensions).await.unwrap();
+					file.read_to_string(&mut buf).unwrap();
-				let mut file = File::open(file.as_std_path()).unwrap();
+					match lex(is(&buf).await, sym!(usercode), &systems, ctx).await {
-				let mut buf = String::new();
+						Ok(lexemes) =>
-				file.read_to_string(&mut buf).unwrap();
+							println!("{}", take_first(&ttv_fmt(&lexemes, &FmtCtxImpl::default()).await, true)),
-				let lexemes = lex(is(&buf).await, sym!(usercode), &systems, ctx).await.unwrap();
+						Err(e) => {
-				let Some(first) = lexemes.first() else {
+							eprintln!("{e}");
-					println!("File empty!");
+							exit_code1.replace(ExitCode::FAILURE);
-					return;
-				};
-				let pctx = HostParseCtxImpl { systems: &systems, ctx: ctx.clone(), src: sym!(usercode) };
-				let snip = Snippet::new(first, &lexemes);
-				match with_reporter(parse_items(&pctx, Substack::Bottom, snip)).await.unwrap() {
-					Err(errv) => {
-						eprintln!("{errv}");
-						*exit_code1.borrow_mut() = ExitCode::FAILURE;
-					},
-					Ok(ptree) if ptree.is_empty() => {
-						eprintln!("File empty only after parsing, but no errors were reported");
-						*exit_code1.borrow_mut() = ExitCode::FAILURE;
-					},
-					Ok(ptree) =>
-						for item in ptree {
-							println!("{}", take_first(&item.print(&FmtCtxImpl::default()).await, true))
 						},
-				};
-			},
-			Commands::Repl => {
-				let mut counter = 0;
-				let mut imports = Vec::new();
-				let usercode_path = sym!(usercode);
-				let mut stdin = BufReader::new(stdin());
-				loop {
-					counter += 1;
-					let (mut root, systems) = init_systems(&args.system, &extensions).await.unwrap();
-					print!("\\.> ");
-					std::io::stdout().flush().unwrap();
-					let mut prompt = String::new();
-					stdin.read_line(&mut prompt).await.unwrap();
-					let name = is(&format!("_{counter}")).await;
-					let path = usercode_path.suffix([name.clone()]).await;
-					let mut lexemes =
-						lex(is(prompt.trim()).await, path.clone(), &systems, ctx).await.unwrap();
-					let Some(discr) = lexemes.first() else { continue };
-					writeln!(
-						log("debug"),
-						"lexed: {}",
-						take_first(&ttv_fmt(&lexemes, &FmtCtxImpl::default()).await, true)
-					)
-					.await;
-					let prefix_sr = SrcRange::zw(path.clone(), 0);
-					let process_lexemes = async |lexemes: &[ParsTokTree]| {
-						let snippet = Snippet::new(&lexemes[0], lexemes);
-						let parse_ctx =
-							HostParseCtxImpl { ctx: ctx.clone(), src: path.clone(), systems: &systems[..] };
-						match try_with_reporter(parse_item(&parse_ctx, Substack::Bottom, vec![], snippet)).await
-						{
-							Ok(items) => Some(items),
-							Err(e) => {
-								eprintln!("{e}");
-								None
-							},
-						}
-					};
-					let add_imports = |items: &mut Vec<Item>, imports: &[Import]| {
-						items.extend(imports.iter().map(|import| Item::new(import.sr.clone(), import.clone())));
-					};
-					if discr.is_kw(is("import").await) {
-						let Some(import_lines) = process_lexemes(&lexemes).await else { continue };
-						imports.extend(import_lines.into_iter().map(|it| match it.kind {
-							ItemKind::Import(imp) => imp,
-							_ => panic!("Expected imports from import line"),
-						}));
-						continue;
 					}
-					if !discr.is_kw(is("let").await) {
+				},
-						let prefix = [is("export").await, is("let").await, name.clone(), is("=").await];
+				Commands::Parse { file } => {
-						lexemes.splice(0..0, prefix.map(|n| Token::Name(n).at(prefix_sr.clone())));
+					let (_, systems) = init_systems(&args.system, &extensions).await.unwrap();
-					}
+					let mut file = File::open(file.as_std_path()).unwrap();
-					let Some(mut new_lines) = process_lexemes(&lexemes).await else { continue };
+					let mut buf = String::new();
-					let const_decl = new_lines.iter().exactly_one().expect("Multiple lines from let");
+					file.read_to_string(&mut buf).unwrap();
-					let input_sr = const_decl.sr.map_range(|_| 0..0);
+					let lexemes = lex(is(&buf).await, sym!(usercode), &systems, ctx).await.unwrap();
-					let const_name = match &const_decl.kind {
+					let Some(first) = lexemes.first() else {
-						ItemKind::Member(ParsedMember { name: const_name, .. }) => const_name.clone(),
+						println!("File empty!");
-						_ => panic!("Expected exactly one constant declaration from let"),
+						return;
 					};
-					add_imports(&mut new_lines, &imports);
+					let pctx = HostParseCtxImpl { systems: &systems, ctx: ctx.clone(), src: sym!(usercode) };
-					imports.push(Import::new(input_sr.clone(), VPath::new(path.segs()), const_name.clone()));
+					let snip = Snippet::new(first, &lexemes);
-					let new_module = ParsedModule::new(true, new_lines);
+					match with_reporter(parse_items(&pctx, Substack::Bottom, snip)).await.unwrap() {
-					match with_reporter(root.add_parsed(&new_module, path.clone())).await {
-						Ok(new) => root = new,
 						Err(errv) => {
 							eprintln!("{errv}");
 							*exit_code1.borrow_mut() = ExitCode::FAILURE;
-							return;
 						},
-					}
+						Ok(ptree) if ptree.is_empty() => {
-					eprintln!("parsed");
+							eprintln!("File empty only after parsing, but no errors were reported");
-					let entrypoint =
-						ExprKind::Const(path.suffix([const_name.clone()]).await).at(input_sr.pos());
-					let mut xctx = ExecCtx::new(root.clone(), entrypoint).await;
-					eprintln!("executed");
-					xctx.set_gas(Some(1000));
-					xctx.execute().await;
-					match xctx.result() {
-						ExecResult::Value(val) => println!(
-							"{const_name} = {}",
-							take_first(&val.print(&FmtCtxImpl::default()).await, false)
-						),
-						ExecResult::Err(e) => println!("error: {e}"),
-						ExecResult::Gas(_) => println!("Ran out of gas!"),
-					}
-				}
-			},
-			Commands::ModTree { proj, prefix } => {
-				let (mut root, _systems) = init_systems(&args.system, &extensions).await.unwrap();
-				if let Some(proj_path) = proj {
-					let path = proj_path.into_std_path_buf();
-					match try_with_reporter(parse_folder(&root, path, sym!(src), ctx.clone())).await {
-						Ok(r) => root = r,
-						Err(e) => {
-							eprintln!("{e}");
 							*exit_code1.borrow_mut() = ExitCode::FAILURE;
-							return;
 						},
-					}
+						Ok(ptree) =>
-				}
+							for item in ptree {
-				let prefix = match prefix {
+								println!("{}", take_first(&item.print(&FmtCtxImpl::default()).await, true))
-					Some(pref) => VPath::parse(&pref).await,
-					None => VPath::new([]),
-				};
-				let root_data = root.0.read().await;
-				print_mod(&root_data.root, prefix, &root_data).await;
-				async fn print_mod(module: &Module, path: VPath, root: &RootData) {
-					let indent = "  ".repeat(path.len());
-					for (key, tgt) in &module.imports {
-						match tgt {
-							Ok(tgt) => println!("{indent}import {key} => {}", tgt.target),
-							Err(opts) => println!(
-								"{indent}import {key} conflicts between {}",
-								opts.iter().map(|i| &i.target).join(" ")
-							),
-						}
-					}
-					for (key, mem) in &module.members {
-						let new_path = path.clone().name_with_suffix(key.clone()).to_sym().await;
-						match mem.kind(root.ctx.clone(), &root.consts).await {
-							MemberKind::Module(module) => {
-								println!("{indent}module {key} {{");
-								print_mod(module, VPath::new(new_path.segs()), root).boxed_local().await;
-								println!("{indent}}}")
 							},
-							MemberKind::Const => {
+					};
-								let value = root.consts.get(&new_path).expect("Missing const!");
+				},
-								println!("{indent}const {key} = {}", fmt(value).await)
+				Commands::Repl => {
-							},
+					let mut counter = 0;
-						}
+					let mut imports = Vec::new();
-					}
+					let usercode_path = sym!(usercode);
-				}
+					let mut stdin = BufReader::new(stdin());
-			},
+					loop {
-			Commands::Exec { proj, code } => {
+						counter += 1;
-				let path = sym!(usercode);
+						let (mut root, systems) = init_systems(&args.system, &extensions).await.unwrap();
-				let prefix_sr = SrcRange::zw(path.clone(), 0);
+						print!("\\.> ");
-				let (mut root, systems) = init_systems(&args.system, &extensions).await.unwrap();
+						std::io::stdout().flush().unwrap();
-				if let Some(proj_path) = proj {
+						let mut prompt = String::new();
-					let path = proj_path.into_std_path_buf();
+						stdin.read_line(&mut prompt).await.unwrap();
-					match try_with_reporter(parse_folder(&root, path, sym!(src), ctx.clone())).await {
+						let name = is(&format!("_{counter}")).await;
-						Ok(r) => root = r,
+						let path = usercode_path.suffix([name.clone()]).await;
-						Err(e) => {
+						let mut lexemes =
-							eprintln!("{e}");
+							lex(is(prompt.trim()).await, path.clone(), &systems, ctx).await.unwrap();
-							*exit_code1.borrow_mut() = ExitCode::FAILURE;
+						let Some(discr) = lexemes.first() else { continue };
-							return;
-						},
-					}
-				}
-				let mut lexemes = match lex(is(code.trim()).await, path.clone(), &systems, ctx).await {
-					Ok(lexemes) => {
 						writeln!(
 							log("debug"),
 							"lexed: {}",
-							fmt_v::<ParsTokTree>(lexemes.iter()).await.join(" ")
+							take_first(&ttv_fmt(&lexemes, &FmtCtxImpl::default()).await, true)
 						)
 						.await;
-						lexemes
+						let prefix_sr = SrcRange::zw(path.clone(), 0);
-					},
+						let process_lexemes = async |lexemes: &[ParsTokTree]| {
-					Err(e) => {
+							let snippet = Snippet::new(&lexemes[0], lexemes);
-						eprintln!("{e}");
+							let parse_ctx =
-						*exit_code1.borrow_mut() = ExitCode::FAILURE;
+								HostParseCtxImpl { ctx: ctx.clone(), src: path.clone(), systems: &systems[..] };
-						return;
+							match try_with_reporter(parse_item(&parse_ctx, Substack::Bottom, vec![], snippet))
-					},
+								.await
-				};
+							{
-				let parse_ctx =
+								Ok(items) => Some(items),
-					HostParseCtxImpl { ctx: ctx.clone(), src: path.clone(), systems: &systems[..] };
+								Err(e) => {
-				let prefix = [is("export").await, is("let").await, is("entrypoint").await, is("=").await];
+									eprintln!("{e}");
-				lexemes.splice(0..0, prefix.map(|n| Token::Name(n).at(prefix_sr.clone())));
+									None
-				let snippet = Snippet::new(&lexemes[0], &lexemes);
+								},
-				let entrypoint = match try_with_reporter(parse_item(
+							}
-					&parse_ctx,
+						};
-					Substack::Bottom,
+						let add_imports = |items: &mut Vec<Item>, imports: &[Import]| {
-					vec![],
+							items
-					snippet,
+								.extend(imports.iter().map(|import| Item::new(import.sr.clone(), import.clone())));
-				))
+						};
-				.await
+						if discr.is_kw(is("import").await) {
-				{
+							let Some(import_lines) = process_lexemes(&lexemes).await else { continue };
-					Ok(items) => ParsedModule::new(true, items),
+							imports.extend(import_lines.into_iter().map(|it| match it.kind {
-					Err(e) => {
+								ItemKind::Import(imp) => imp,
-						eprintln!("{e}");
+								_ => panic!("Expected imports from import line"),
-						*exit_code1.borrow_mut() = ExitCode::FAILURE;
+							}));
-						return;
+							continue;
-					},
+						}
-				};
+						if !discr.is_kw(is("let").await) {
-				let root = match with_reporter(root.add_parsed(&entrypoint, path.clone())).await {
+							let prefix = [is("export").await, is("let").await, name.clone(), is("=").await];
-					Err(e) => {
+							lexemes.splice(0..0, prefix.map(|n| Token::Name(n).at(prefix_sr.clone())));
-						eprintln!("{e}");
+						}
-						*exit_code1.borrow_mut() = ExitCode::FAILURE;
+						let Some(mut new_lines) = process_lexemes(&lexemes).await else { continue };
-						return;
+						let const_decl = new_lines.iter().exactly_one().expect("Multiple lines from let");
-					},
+						let input_sr = const_decl.sr.map_range(|_| 0..0);
-					Ok(new_root) => new_root,
+						let const_name = match &const_decl.kind {
-				};
+							ItemKind::Member(ParsedMember { name: const_name, .. }) => const_name.clone(),
-				let expr = ExprKind::Const(sym!(usercode::entrypoint)).at(prefix_sr.pos());
+							_ => panic!("Expected exactly one constant declaration from let"),
-				let mut xctx = ExecCtx::new(root, expr).await;
+						};
-				xctx.set_gas(Some(10_000));
+						add_imports(&mut new_lines, &imports);
-				xctx.execute().await;
+						imports.push(Import::new(
-				match xctx.result() {
+							input_sr.clone(),
-					ExecResult::Value(val) =>
+							VPath::new(path.segs()),
-						println!("{}", take_first(&val.print(&FmtCtxImpl::default()).await, false)),
+							const_name.clone(),
-					ExecResult::Err(e) => println!("error: {e}"),
+						));
-					ExecResult::Gas(_) => println!("Ran out of gas!"),
+						let new_module = ParsedModule::new(true, new_lines);
-				}
+						match with_reporter(root.add_parsed(&new_module, path.clone())).await {
-			},
+							Ok(new) => root = new,
+							Err(errv) => {
+								eprintln!("{errv}");
+								*exit_code1.borrow_mut() = ExitCode::FAILURE;
+								return;
+							},
+						}
+						eprintln!("parsed");
+						let entrypoint =
+							ExprKind::Const(path.suffix([const_name.clone()]).await).at(input_sr.pos());
+						let mut xctx = ExecCtx::new(root.clone(), entrypoint).await;
+						eprintln!("executed");
+						xctx.set_gas(Some(1000));
+						xctx.execute().await;
+						match xctx.result() {
+							ExecResult::Value(val) => println!(
+								"{const_name} = {}",
+								take_first(&val.print(&FmtCtxImpl::default()).await, false)
+							),
+							ExecResult::Err(e) => println!("error: {e}"),
+							ExecResult::Gas(_) => println!("Ran out of gas!"),
+						}
+					}
+				},
+				Commands::ModTree { proj, prefix } => {
+					let (mut root, _systems) = init_systems(&args.system, &extensions).await.unwrap();
+					if let Some(proj_path) = proj {
+						let path = proj_path.into_std_path_buf();
+						match try_with_reporter(parse_folder(&root, path, sym!(src), ctx.clone())).await {
+							Ok(r) => root = r,
+							Err(e) => {
+								eprintln!("{e}");
+								*exit_code1.borrow_mut() = ExitCode::FAILURE;
+								return;
+							},
+						}
+					}
+					let prefix = match prefix {
+						Some(pref) => VPath::parse(&pref).await,
+						None => VPath::new([]),
+					};
+					let root_data = root.0.read().await;
+					print_mod(&root_data.root, prefix, &root_data).await;
+					async fn print_mod(module: &Module, path: VPath, root: &RootData) {
+						let indent = "  ".repeat(path.len());
+						for (key, tgt) in &module.imports {
+							match tgt {
+								Ok(tgt) => println!("{indent}import {key} => {}", tgt.target),
+								Err(opts) => println!(
+									"{indent}import {key} conflicts between {}",
+									opts.iter().map(|i| &i.target).join(" ")
+								),
+							}
+						}
+						for (key, mem) in &module.members {
+							let new_path = path.clone().name_with_suffix(key.clone()).to_sym().await;
+							match mem.kind(root.ctx.clone(), &root.consts).await {
+								MemberKind::Module(module) => {
+									println!("{indent}module {key} {{");
+									print_mod(module, VPath::new(new_path.segs()), root).boxed_local().await;
+									println!("{indent}}}")
+								},
+								MemberKind::Const => {
+									let value = root.consts.get(&new_path).expect("Missing const!");
+									println!("{indent}const {key} = {}", fmt(value).await)
+								},
+							}
+						}
+					}
+				},
+				Commands::Exec { proj, code } => {
+					let path = sym!(usercode);
+					let prefix_sr = SrcRange::zw(path.clone(), 0);
+					let (mut root, systems) = init_systems(&args.system, &extensions).await.unwrap();
+					if let Some(proj_path) = proj {
+						let path = proj_path.into_std_path_buf();
+						match try_with_reporter(parse_folder(&root, path, sym!(src), ctx.clone())).await {
+							Ok(r) => root = r,
+							Err(e) => {
+								eprintln!("{e}");
+								*exit_code1.borrow_mut() = ExitCode::FAILURE;
+								return;
+							},
+						}
+					}
+					let mut lexemes = match lex(is(code.trim()).await, path.clone(), &systems, ctx).await {
+						Ok(lexemes) => {
+							writeln!(
+								log("debug"),
+								"lexed: {}",
+								fmt_v::<ParsTokTree>(lexemes.iter()).await.join(" ")
+							)
+							.await;
+							lexemes
+						},
+						Err(e) => {
+							eprintln!("{e}");
+							*exit_code1.borrow_mut() = ExitCode::FAILURE;
+							return;
+						},
+					};
+					let parse_ctx =
+						HostParseCtxImpl { ctx: ctx.clone(), src: path.clone(), systems: &systems[..] };
+					let prefix = [is("export").await, is("let").await, is("entrypoint").await, is("=").await];
+					lexemes.splice(0..0, prefix.map(|n| Token::Name(n).at(prefix_sr.clone())));
+					let snippet = Snippet::new(&lexemes[0], &lexemes);
+					let entrypoint = match try_with_reporter(parse_item(
+						&parse_ctx,
+						Substack::Bottom,
+						vec![],
+						snippet,
+					))
+					.await
+					{
+						Ok(items) => ParsedModule::new(true, items),
+						Err(e) => {
+							eprintln!("{e}");
+							*exit_code1.borrow_mut() = ExitCode::FAILURE;
+							return;
+						},
+					};
+					let root = match with_reporter(root.add_parsed(&entrypoint, path.clone())).await {
+						Err(e) => {
+							eprintln!("{e}");
+							*exit_code1.borrow_mut() = ExitCode::FAILURE;
+							return;
+						},
+						Ok(new_root) => new_root,
+					};
+					let expr = ExprKind::Const(sym!(usercode::entrypoint)).at(prefix_sr.pos());
+					let mut xctx = ExecCtx::new(root.clone(), expr).await;
+					xctx.set_gas(Some(10_000));
+					xctx.execute().await;
+					match xctx.result() {
+						ExecResult::Value(val) => {
+							println!("{}", take_first(&val.print(&FmtCtxImpl::default()).await, false))
+						},
+						ExecResult::Err(e) => println!("error: {e}"),
+						ExecResult::Gas(_) => println!("Ran out of gas!"),
+					}
+				},
+			};
+		})
+		.await;
+		signal_end_main.send(()).expect("cleanup should still be waiting");
+	});
+	let cleanup = async {
+		if on_end_main.await.is_err() {
+			return;
 		}
-	}));
+		tokio::time::sleep(Duration::from_secs(2)).await;
-	with_interner(local_interner(), with_logger(logger, local_set)).await;
+		let mut extensions = HashMap::new();
+		let systems = ctx.systems.read().await.values().filter_map(|v| v.upgrade()).collect_vec();
+		let exprs = ctx.exprs.iter().collect_vec();
+		for system in &systems {
+			extensions.insert(system.ext().name().clone(), system.ext().clone());
+		}
+		if extensions.is_empty() && systems.is_empty() && exprs.is_empty() {
+			return;
+		}
+		eprintln!("Shutdown is taking long. The following language constructs are still live:");
+		eprintln!("Extensions: {}", extensions.keys().join(", "));
+		for sys in &systems {
+			eprintln!("System: {:?} = {}", sys.id(), sys.ctor().name())
+		}
+		for (rc, expr) in &exprs {
+			eprintln!("{rc}x {:?} = {}", expr.id(), fmt(expr).await)
+		}
+		std::process::abort()
+	};
+	futures::future::select(
+		pin!(cleanup),
+		pin!(with_interner(local_interner(), with_logger(logger, local_set))),
+	)
+	.await;
 	let x = *exit_code.borrow();
 	Ok(x)
 }

xtask/src/main.rs

@@ -23,6 +23,8 @@ pub struct Args {
 pub enum Commands {
 	CheckApiRefs,
 	Orcx {
+		#[arg(long)]
+		release: bool,
 		#[arg(trailing_var_arg = true, num_args = 1..)]
 		argv: Vec<String>,
 	},
@@ -41,7 +43,7 @@ fn main() -> io::Result<ExitCode> {
 	let args = Args::parse();
 	match &args.command {
 		Commands::CheckApiRefs => check_api_refs(&args)?,
-		Commands::Orcx { argv } => orcx(&args, argv)?,
+		Commands::Orcx { release, argv } => orcx(*release, &args, argv)?,
 		Commands::Orcxdb { argv } => orcxdb(&args, argv)?,
 	}
 	Ok(if EXIT_OK.load(Ordering::Relaxed) { ExitCode::SUCCESS } else { ExitCode::FAILURE })

xtask/src/orcx.rs

@@ -4,17 +4,25 @@ use std::sync::atomic::Ordering;
 
 use crate::{Args, EXIT_OK};
 
-pub fn orcx(_args: &Args, argv: &[String]) -> io::Result<()> {
+pub fn orcx(release: bool, _args: &Args, argv: &[String]) -> io::Result<()> {
-	if !Command::new("cargo").args(["build", "-p", "orchid-std", "--quiet"]).status()?.success() {
+	let mut std_build_cmd = Command::new("cargo");
+	std_build_cmd.args(["build", "-p", "orchid-std", "--quiet"]);
+	if release {
+		std_build_cmd.arg("--release");
+	}
+
+	if !std_build_cmd.status()?.success() {
 		EXIT_OK.store(false, Ordering::Relaxed);
 		return Ok(());
 	}
-	if !Command::new("cargo")
+	let mut run_cmd = Command::new("cargo");
-		.args(["run", "-p", "orcx", "--quiet", "--"])
+	run_cmd.args(["run", "-p", "orcx", "--quiet"]);
-		.args(argv)
+	if release {
-		.status()?
+		run_cmd.arg("--release");
-		.success()
+	}
-	{
+	run_cmd.arg("--");
+	run_cmd.args(argv);
+	if !run_cmd.status()?.success() {
 		EXIT_OK.store(false, Ordering::Relaxed);
 	}
 	Ok(())