From 2b79e96dc90ae85adece22d9f5930b824c4d779d Mon Sep 17 00:00:00 2001 From: Lawrence Bethlenfalvy Date: Sat, 25 Jan 2025 00:10:49 +0100 Subject: [PATCH] base and extension fully compiles, host in good shape --- Cargo.lock | 115 ++++++ SWAP.md | 6 +- orchid-api-derive/src/decode.rs | 22 +- orchid-api-derive/src/encode.rs | 6 +- orchid-api-traits/Cargo.toml | 3 + orchid-api-traits/src/coding.rs | 224 +++++++----- orchid-api-traits/src/helpers.rs | 26 +- orchid-api-traits/src/relations.rs | 11 +- orchid-api/Cargo.toml | 4 + orchid-api/src/atom.rs | 7 + orchid-api/src/interner.rs | 1 - orchid-api/src/parser.rs | 11 +- orchid-api/src/proto.rs | 71 ++-- orchid-base/Cargo.toml | 1 + orchid-base/src/builtin.rs | 8 +- orchid-base/src/format.rs | 89 +++++ orchid-base/src/interner.rs | 13 +- orchid-base/src/lib.rs | 1 + orchid-base/src/macros.rs | 20 +- orchid-base/src/msg.rs | 4 +- orchid-base/src/parse.rs | 5 +- orchid-base/src/reqnot.rs | 20 +- orchid-base/src/tree.rs | 116 +++--- orchid-extension/src/atom.rs | 48 +-- orchid-extension/src/atom_owned.rs | 48 +-- orchid-extension/src/atom_thin.rs | 48 +-- orchid-extension/src/conv.rs | 2 +- orchid-extension/src/entrypoint.rs | 39 +- orchid-extension/src/expr.rs | 14 +- orchid-extension/src/func_atom.rs | 9 +- orchid-extension/src/macros.rs | 2 +- orchid-extension/src/system.rs | 21 +- orchid-host/Cargo.toml | 3 + orchid-host/src/atom.rs | 98 +++++ orchid-host/src/child.rs | 44 --- orchid-host/src/ctx.rs | 46 +++ orchid-host/src/expr.rs | 104 +++--- orchid-host/src/expr_store.rs | 35 ++ orchid-host/src/extension.rs | 566 ++++++++++------------------- orchid-host/src/lex.rs | 107 +++--- orchid-host/src/lib.rs | 5 +- orchid-host/src/macros.rs | 65 ++-- orchid-host/src/parse.rs | 262 ++++++------- orchid-host/src/rule/build.rs | 60 +-- orchid-host/src/rule/matcher.rs | 22 +- orchid-host/src/rule/scal_match.rs | 2 +- orchid-host/src/subprocess.rs | 117 +++--- orchid-host/src/system.rs | 213 +++++++++++ orchid-host/src/tree.rs | 123 ++++--- 49 files changed, 1719 insertions(+), 1168 deletions(-) create mode 100644 orchid-base/src/format.rs create mode 100644 orchid-host/src/atom.rs delete mode 100644 orchid-host/src/child.rs create mode 100644 orchid-host/src/ctx.rs create mode 100644 orchid-host/src/expr_store.rs create mode 100644 orchid-host/src/system.rs diff --git a/Cargo.lock b/Cargo.lock index 5f341eb..fd505a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -174,6 +183,43 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288f83726785267c6f2ef073a3d83dc3f9b81464e9f99898240cced85fce35a" +[[package]] +name = "async-process" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" +dependencies = [ + "async-channel 2.3.1", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener 5.4.0", + "futures-lite", + "rustix", + "tracing", +] + +[[package]] +name = "async-signal" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys", +] + [[package]] name = "async-std" version = "1.13.0" @@ -200,6 +246,28 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2 1.0.92", + "quote 1.0.38", + "syn 2.0.95", +] + [[package]] name = "async-task" version = "4.7.1" @@ -895,9 +963,11 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" name = "orchid-api" version = "0.1.0" dependencies = [ + "async-std", "orchid-api-derive", "orchid-api-traits", "ordered-float", + "test_executors", ] [[package]] @@ -916,6 +986,9 @@ dependencies = [ name = "orchid-api-traits" version = "0.1.0" dependencies = [ + "async-std", + "async-stream", + "futures", "itertools", "never", "ordered-float", @@ -939,6 +1012,7 @@ dependencies = [ "orchid-api-derive", "orchid-api-traits", "ordered-float", + "regex", "rust-embed", "rust_decimal", "some_executor 0.4.0", @@ -978,7 +1052,9 @@ dependencies = [ name = "orchid-host" version = "0.1.0" dependencies = [ + "async-process", "async-std", + "async-stream", "derive_destructure", "futures", "hashbrown 0.15.2", @@ -992,6 +1068,7 @@ dependencies = [ "ordered-float", "paste", "substack", + "test_executors", "trait-set", ] @@ -1196,6 +1273,35 @@ dependencies = [ "getrandom", ] +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + [[package]] name = "rend" version = "0.4.2" @@ -1361,6 +1467,15 @@ dependencies = [ "digest", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "simdutf8" version = "0.1.5" diff --git a/SWAP.md b/SWAP.md index 55ff426..f74cbe1 100644 --- a/SWAP.md +++ b/SWAP.md @@ -2,7 +2,11 @@ convert host to async non-send -convert extension's SysCtx to a typed context bag +demonstrate operation with existing lex-hello example + +consider converting extension's SysCtx to a typed context bag + +align fn atom and macros on both sides with new design. No global state. ## alternate extension mechanism diff --git a/orchid-api-derive/src/decode.rs b/orchid-api-derive/src/decode.rs index 3ca553e..04f19af 100644 --- a/orchid-api-derive/src/decode.rs +++ b/orchid-api-derive/src/decode.rs @@ -12,7 +12,9 @@ pub fn derive(input: TokenStream) -> TokenStream { let decode = decode_body(&input.data); let expanded = quote! { impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause { - fn decode(read: &mut R) -> Self { #decode } + async fn decode(mut read: std::pin::Pin<&mut R>) -> Self { + #decode + } } }; TokenStream::from(expanded) @@ -22,11 +24,21 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream { match fields { syn::Fields::Unit => quote! {}, syn::Fields::Named(_) => { - let names = fields.iter().map(|f| f.ident.as_ref().unwrap()); - quote! { { #( #names: orchid_api_traits::Decode::decode(read), )* } } + let exprs = fields.iter().map(|f| { + let syn::Field { ty, ident, .. } = &f; + quote! { + #ident : < #ty as orchid_api_traits::Decode>::decode(read.as_mut()).await + } + }); + quote! { { #( #exprs, )* } } }, syn::Fields::Unnamed(_) => { - let exprs = fields.iter().map(|_| quote! { orchid_api_traits::Decode::decode(read), }); + let exprs = fields.iter().map(|field| { + let ty = &field.ty; + quote! { + < #ty as orchid_api_traits::Decode>::decode(read.as_mut()).await, + } + }); quote! { ( #( #exprs )* ) } }, } @@ -46,7 +58,7 @@ fn decode_body(data: &syn::Data) -> proc_macro2::TokenStream { quote! { #id => Self::#ident #fields, } }); quote! { - match ::decode(read) { + match ::decode(read.as_mut()).await { #(#opts)* x => panic!("Unrecognized enum kind {x}") } diff --git a/orchid-api-derive/src/encode.rs b/orchid-api-derive/src/encode.rs index fd826db..cf3d82d 100644 --- a/orchid-api-derive/src/encode.rs +++ b/orchid-api-derive/src/encode.rs @@ -14,7 +14,7 @@ pub fn derive(input: TokenStream) -> TokenStream { let encode = encode_body(&input.data); let expanded = quote! { impl #e_impl_generics orchid_api_traits::Encode for #name #e_ty_generics #e_where_clause { - fn encode(&self, write: &mut W) { #encode } + async fn encode(&self, mut write: std::pin::Pin<&mut W>) { #encode } } }; TokenStream::from(expanded) @@ -37,7 +37,7 @@ fn encode_body(data: &syn::Data) -> Option { let body = encode_items(&v.fields); quote! { Self::#ident #dest => { - (#i as u8).encode(write); + (#i as u8).encode(write.as_mut()).await; #body } } @@ -53,7 +53,7 @@ fn encode_body(data: &syn::Data) -> Option { } fn encode_names(names: impl Iterator) -> pm2::TokenStream { - quote! { #( #names .encode(write); )* } + quote! { #( #names .encode(write.as_mut()).await; )* } } fn encode_items(fields: &syn::Fields) -> Option { diff --git a/orchid-api-traits/Cargo.toml b/orchid-api-traits/Cargo.toml index cee1a24..6712518 100644 --- a/orchid-api-traits/Cargo.toml +++ b/orchid-api-traits/Cargo.toml @@ -6,6 +6,9 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-std = "1.13.0" +async-stream = "0.3.6" +futures = "0.3.31" itertools = "0.14.0" never = "0.1.0" ordered-float = "4.6.0" diff --git a/orchid-api-traits/src/coding.rs b/orchid-api-traits/src/coding.rs index 4fa6769..1fea9dd 100644 --- a/orchid-api-traits/src/coding.rs +++ b/orchid-api-traits/src/coding.rs @@ -1,29 +1,38 @@ -use std::borrow::Cow; use std::collections::HashMap; +use std::future::Future; use std::hash::Hash; -use std::io::{Read, Write}; -use std::iter; +use std::num::NonZero; use std::ops::{Range, RangeInclusive}; +use std::pin::Pin; use std::rc::Rc; use std::sync::Arc; +use async_std::io::{Read, ReadExt, Write, WriteExt}; +use async_stream::stream; +use futures::future::LocalBoxFuture; +use futures::{FutureExt, StreamExt}; use never::Never; use ordered_float::NotNan; use crate::encode_enum; -pub trait Decode { +pub trait Decode: 'static { /// Decode an instance from the beginning of the buffer. Return the decoded /// data and the remaining buffer. - fn decode(read: &mut R) -> Self; + fn decode(read: Pin<&mut R>) -> impl Future + '_; } pub trait Encode { /// Append an instance of the struct to the buffer - fn encode(&self, write: &mut W); + fn encode(&self, write: Pin<&mut W>) -> impl Future; } pub trait Coding: Encode + Decode + Clone { - fn get_decoder(map: impl Fn(Self) -> T + 'static) -> impl Fn(&mut dyn Read) -> T { - move |r| map(Self::decode(r)) + fn get_decoder + 'static>( + map: impl Fn(Self) -> F + Clone + 'static, + ) -> impl for<'a> Fn(Pin<&'a mut dyn Read>) -> LocalBoxFuture<'a, T> { + move |r| { + let map = map.clone(); + async move { map(Self::decode(r).await).await }.boxed_local() + } } } impl Coding for T {} @@ -31,15 +40,15 @@ impl Coding for T {} macro_rules! num_impl { ($number:ty) => { impl Decode for $number { - fn decode(read: &mut R) -> Self { + async fn decode(mut read: Pin<&mut R>) -> Self { let mut bytes = [0u8; (<$number>::BITS / 8) as usize]; - read.read_exact(&mut bytes).unwrap(); + read.read_exact(&mut bytes).await.unwrap(); <$number>::from_be_bytes(bytes) } } impl Encode for $number { - fn encode(&self, write: &mut W) { - write.write_all(&self.to_be_bytes()).expect("Could not write number") + async fn encode(&self, mut write: Pin<&mut W>) { + write.write_all(&self.to_be_bytes()).await.expect("Could not write number") } } }; @@ -57,41 +66,45 @@ num_impl!(i8); macro_rules! nonzero_impl { ($name:ty) => { - impl Decode for $name { - fn decode(read: &mut R) -> Self { Self::new(Decode::decode(read)).unwrap() } + impl Decode for NonZero<$name> { + async fn decode(read: Pin<&mut R>) -> Self { + Self::new(<$name as Decode>::decode(read).await).unwrap() + } } - impl Encode for $name { - fn encode(&self, write: &mut W) { self.get().encode(write) } + impl Encode for NonZero<$name> { + async fn encode(&self, write: Pin<&mut W>) { + self.get().encode(write).await + } } }; } -nonzero_impl!(std::num::NonZeroU8); -nonzero_impl!(std::num::NonZeroU16); -nonzero_impl!(std::num::NonZeroU32); -nonzero_impl!(std::num::NonZeroU64); -nonzero_impl!(std::num::NonZeroU128); -nonzero_impl!(std::num::NonZeroI8); -nonzero_impl!(std::num::NonZeroI16); -nonzero_impl!(std::num::NonZeroI32); -nonzero_impl!(std::num::NonZeroI64); -nonzero_impl!(std::num::NonZeroI128); +nonzero_impl!(u8); +nonzero_impl!(u16); +nonzero_impl!(u32); +nonzero_impl!(u64); +nonzero_impl!(u128); +nonzero_impl!(i8); +nonzero_impl!(i16); +nonzero_impl!(i32); +nonzero_impl!(i64); +nonzero_impl!(i128); impl Encode for &T { - fn encode(&self, write: &mut W) { (**self).encode(write) } + async fn encode(&self, write: Pin<&mut W>) { (**self).encode(write).await } } macro_rules! float_impl { ($t:ty, $size:expr) => { impl Decode for NotNan<$t> { - fn decode(read: &mut R) -> Self { + async fn decode(mut read: Pin<&mut R>) -> Self { let mut bytes = [0u8; $size]; - read.read_exact(&mut bytes).unwrap(); + read.read_exact(&mut bytes).await.unwrap(); NotNan::new(<$t>::from_be_bytes(bytes)).expect("Float was NaN") } } impl Encode for NotNan<$t> { - fn encode(&self, write: &mut W) { - write.write_all(&self.as_ref().to_be_bytes()).expect("Could not write number") + async fn encode(&self, mut write: Pin<&mut W>) { + write.write_all(&self.as_ref().to_be_bytes()).await.expect("Could not write number") } } }; @@ -101,98 +114,103 @@ float_impl!(f64, 8); float_impl!(f32, 4); impl Decode for String { - fn decode(read: &mut R) -> Self { - let len = u64::decode(read).try_into().unwrap(); + async fn decode(mut read: Pin<&mut R>) -> Self { + let len = u64::decode(read.as_mut()).await.try_into().unwrap(); let mut data = vec![0u8; len]; - read.read_exact(&mut data).unwrap(); + read.read_exact(&mut data).await.unwrap(); std::str::from_utf8(&data).expect("String invalid UTF-8").to_owned() } } impl Encode for String { - fn encode(&self, write: &mut W) { - u64::try_from(self.len()).unwrap().encode(write); - write.write_all(self.as_bytes()).unwrap() + async fn encode(&self, mut write: Pin<&mut W>) { + u64::try_from(self.len()).unwrap().encode(write.as_mut()).await; + write.write_all(self.as_bytes()).await.unwrap() } } impl Encode for str { - fn encode(&self, write: &mut W) { - u64::try_from(self.len()).unwrap().encode(write); - write.write_all(self.as_bytes()).unwrap() + async fn encode(&self, mut write: Pin<&mut W>) { + u64::try_from(self.len()).unwrap().encode(write.as_mut()).await; + write.write_all(self.as_bytes()).await.unwrap() } } impl Decode for Vec { - fn decode(read: &mut R) -> Self { - let len = u64::decode(read).try_into().unwrap(); - iter::repeat_with(|| T::decode(read)).take(len).collect() + async fn decode(mut read: Pin<&mut R>) -> Self { + let len = u64::decode(read.as_mut()).await.try_into().unwrap(); + stream! { loop { yield T::decode(read.as_mut()).await } }.take(len).collect().await } } impl Encode for Vec { - fn encode(&self, write: &mut W) { - u64::try_from(self.len()).unwrap().encode(write); - self.iter().for_each(|t| t.encode(write)); + async fn encode(&self, write: Pin<&mut W>) { + self.as_slice().encode(write).await } } impl Encode for [T] { - fn encode(&self, write: &mut W) { - u64::try_from(self.len()).unwrap().encode(write); - self.iter().for_each(|t| t.encode(write)); + async fn encode(&self, mut write: Pin<&mut W>) { + u64::try_from(self.len()).unwrap().encode(write.as_mut()).await; + for t in self.iter() { + t.encode(write.as_mut()).await + } } } impl Decode for Option { - fn decode(read: &mut R) -> Self { - match u8::decode(read) { + async fn decode(mut read: Pin<&mut R>) -> Self { + match u8::decode(read.as_mut()).await { 0 => None, - 1 => Some(T::decode(read)), + 1 => Some(T::decode(read).await), x => panic!("{x} is not a valid option value"), } } } impl Encode for Option { - fn encode(&self, write: &mut W) { - let t = if let Some(t) = self { t } else { return 0u8.encode(write) }; - 1u8.encode(write); - t.encode(write); + async fn encode(&self, mut write: Pin<&mut W>) { + let t = if let Some(t) = self { t } else { return 0u8.encode(write.as_mut()).await }; + 1u8.encode(write.as_mut()).await; + t.encode(write).await; } } impl Decode for Result { - fn decode(read: &mut R) -> Self { - match u8::decode(read) { - 0 => Self::Ok(T::decode(read)), - 1 => Self::Err(E::decode(read)), + async fn decode(mut read: Pin<&mut R>) -> Self { + match u8::decode(read.as_mut()).await { + 0 => Self::Ok(T::decode(read).await), + 1 => Self::Err(E::decode(read).await), x => panic!("Invalid Result tag {x}"), } } } impl Encode for Result { - fn encode(&self, write: &mut W) { + async fn encode(&self, write: Pin<&mut W>) { match self { - Ok(t) => encode_enum(write, 0, |w| t.encode(w)), - Err(e) => encode_enum(write, 1, |w| e.encode(w)), + Ok(t) => encode_enum(write, 0, |w| t.encode(w)).await, + Err(e) => encode_enum(write, 1, |w| e.encode(w)).await, } } } impl Decode for HashMap { - fn decode(read: &mut R) -> Self { - let len = u64::decode(read).try_into().unwrap(); - iter::repeat_with(|| <(K, V)>::decode(read)).take(len).collect() + async fn decode(mut read: Pin<&mut R>) -> Self { + let len = u64::decode(read.as_mut()).await.try_into().unwrap(); + stream! { loop { yield <(K, V)>::decode(read.as_mut()).await } }.take(len).collect().await } } impl Encode for HashMap { - fn encode(&self, write: &mut W) { - u64::try_from(self.len()).unwrap().encode(write); - self.iter().for_each(|pair| pair.encode(write)); + async fn encode(&self, mut write: Pin<&mut W>) { + u64::try_from(self.len()).unwrap().encode(write.as_mut()).await; + for pair in self.iter() { + pair.encode(write.as_mut()).await + } } } macro_rules! tuple { (($($t:ident)*) ($($T:ident)*)) => { impl<$($T: Decode),*> Decode for ($($T,)*) { - fn decode(read: &mut R) -> Self { ($($T::decode(read),)*) } + async fn decode(mut read: Pin<&mut R>) -> Self { + ($($T::decode(read.as_mut()).await,)*) + } } impl<$($T: Encode),*> Encode for ($($T,)*) { - fn encode(&self, write: &mut W) { + async fn encode(&self, mut write: Pin<&mut W>) { let ($($t,)*) = self; - $( $t.encode(write); )* + $( $t.encode(write.as_mut()).await; )* } } }; @@ -216,52 +234,59 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I)); tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16 impl Decode for () { - fn decode(_: &mut R) -> Self {} + async fn decode(_: Pin<&mut R>) -> Self {} } impl Encode for () { - fn encode(&self, _: &mut W) {} + async fn encode(&self, _: Pin<&mut W>) {} } impl Decode for Never { - fn decode(_: &mut R) -> Self { + async fn decode(_: Pin<&mut R>) -> Self { unreachable!("A value of Never cannot exist so it can't have been serialized"); } } impl Encode for Never { - fn encode(&self, _: &mut W) { match *self {} } + async fn encode(&self, _: Pin<&mut W>) { match *self {} } } impl Decode for bool { - fn decode(read: &mut R) -> Self { + async fn decode(mut read: Pin<&mut R>) -> Self { let mut buf = [0]; - read.read_exact(&mut buf).unwrap(); + read.read_exact(&mut buf).await.unwrap(); buf[0] != 0 } } impl Encode for bool { - fn encode(&self, write: &mut W) { - write.write_all(&[if *self { 0xff } else { 0 }]).unwrap() + async fn encode(&self, mut write: Pin<&mut W>) { + write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await.unwrap() } } impl Decode for [T; N] { - fn decode(read: &mut R) -> Self { + async fn decode(mut read: Pin<&mut R>) -> Self { // TODO: figure out how to do this in safe rust on the stack - ((0..N).map(|_| T::decode(read)).collect::>().try_into()) - .unwrap_or_else(|_| unreachable!("The length of this iterator is statically known")) + let v = + stream! { loop { yield T::decode(read.as_mut()).await } }.take(N).collect::>().await; + v.try_into().unwrap_or_else(|_| unreachable!("The length of this stream is statically known")) } } impl Encode for [T; N] { - fn encode(&self, write: &mut W) { self.iter().for_each(|t| t.encode(write)) } + async fn encode(&self, mut write: Pin<&mut W>) { + for t in self.iter() { + t.encode(write.as_mut()).await + } + } } macro_rules! two_end_range { ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => { impl Decode for $name { - fn decode(read: &mut R) -> Self { T::decode(read) $op T::decode(read) } + async fn decode(mut read: Pin<&mut R>) -> Self { + T::decode(read.as_mut()).await $op T::decode(read).await + } } impl Encode for $name { - fn encode(&self, write: &mut W) { + async fn encode(&self, mut write: Pin<&mut W>) { let $this = self; - ($start).encode(write); - ($end).encode(write); + ($start).encode(write.as_mut()).await; + ($end).encode(write).await; } } } @@ -273,10 +298,12 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end()); macro_rules! smart_ptr { ($name:tt) => { impl Decode for $name { - fn decode(read: &mut R) -> Self { $name::new(T::decode(read)) } + async fn decode(read: Pin<&mut R>) -> Self { + $name::new(T::decode(read).await) + } } impl Encode for $name { - fn encode(&self, write: &mut W) { (**self).encode(write) } + async fn encode(&self, write: Pin<&mut W>) { (**self).encode(write).await } } }; } @@ -285,18 +312,13 @@ smart_ptr!(Arc); smart_ptr!(Rc); smart_ptr!(Box); -impl Decode for Cow<'_, T> -where T::Owned: Decode -{ - fn decode(read: &mut R) -> Self { Cow::Owned(T::Owned::decode(read)) } -} -impl Encode for Cow<'_, T> { - fn encode(&self, write: &mut W) { (**self).encode(write) } -} - impl Decode for char { - fn decode(read: &mut R) -> Self { char::from_u32(u32::decode(read)).unwrap() } + async fn decode(read: Pin<&mut R>) -> Self { + char::from_u32(u32::decode(read).await).unwrap() + } } impl Encode for char { - fn encode(&self, write: &mut W) { (*self as u32).encode(write) } + async fn encode(&self, write: Pin<&mut W>) { + (*self as u32).encode(write).await + } } diff --git a/orchid-api-traits/src/helpers.rs b/orchid-api-traits/src/helpers.rs index 69a7020..5adc319 100644 --- a/orchid-api-traits/src/helpers.rs +++ b/orchid-api-traits/src/helpers.rs @@ -1,16 +1,22 @@ -use std::io::{Read, Write}; +use std::future::Future; +use std::pin::Pin; +use async_std::io::{Read, ReadExt, Write, WriteExt}; use itertools::{Chunk, Itertools}; use crate::Encode; -pub fn encode_enum(write: &mut W, id: u8, f: impl FnOnce(&mut W)) { - id.encode(write); - f(write) +pub async fn encode_enum<'a, W: Write + ?Sized, F: Future>( + mut write: Pin<&'a mut W>, + id: u8, + f: impl FnOnce(Pin<&'a mut W>) -> F, +) { + id.encode(write.as_mut()).await; + f(write).await } -pub fn write_exact(write: &mut W, bytes: &'static [u8]) { - write.write_all(bytes).expect("Failed to write exact bytes") +pub async fn write_exact(mut write: Pin<&mut W>, bytes: &'static [u8]) { + write.write_all(bytes).await.expect("Failed to write exact bytes") } pub fn print_bytes(b: &[u8]) -> String { @@ -21,16 +27,16 @@ pub fn print_bytes(b: &[u8]) -> String { .join(" ") } -pub fn read_exact(read: &mut R, bytes: &'static [u8]) { +pub async fn read_exact(mut read: Pin<&mut R>, bytes: &'static [u8]) { let mut data = vec![0u8; bytes.len()]; - read.read_exact(&mut data).expect("Failed to read bytes"); + read.read_exact(&mut data).await.expect("Failed to read bytes"); if data != bytes { panic!("Wrong bytes!\nExpected: {}\nFound: {}", print_bytes(bytes), print_bytes(&data)); } } -pub fn enc_vec(enc: &impl Encode) -> Vec { +pub async fn enc_vec(enc: &impl Encode) -> Vec { let mut vec = Vec::new(); - enc.encode(&mut vec); + enc.encode(Pin::new(&mut vec)).await; vec } diff --git a/orchid-api-traits/src/relations.rs b/orchid-api-traits/src/relations.rs index 6a505f8..4caf5c4 100644 --- a/orchid-api-traits/src/relations.rs +++ b/orchid-api-traits/src/relations.rs @@ -1,3 +1,5 @@ +use std::future::Future; + use super::coding::Coding; use crate::helpers::enc_vec; @@ -5,9 +7,12 @@ pub trait Request: Coding + Sized + Send + 'static { type Response: Coding + Send + 'static; } -pub fn respond(_: &R, rep: R::Response) -> Vec { enc_vec(&rep) } -pub fn respond_with(r: &R, f: impl FnOnce(&R) -> R::Response) -> Vec { - respond(r, f(r)) +pub async fn respond(_: &R, rep: R::Response) -> Vec { enc_vec(&rep).await } +pub async fn respond_with>( + r: &R, + f: impl FnOnce(&R) -> F, +) -> Vec { + respond(r, f(r).await).await } pub trait Channel: 'static { diff --git a/orchid-api/Cargo.toml b/orchid-api/Cargo.toml index ccb5342..ea2b2b0 100644 --- a/orchid-api/Cargo.toml +++ b/orchid-api/Cargo.toml @@ -9,3 +9,7 @@ edition = "2021" ordered-float = "4.6.0" orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" } orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" } +async-std = "1.13.0" + +[dev-dependencies] +test_executors = "0.3.2" diff --git a/orchid-api/src/atom.rs b/orchid-api/src/atom.rs index a1ca143..87ccd0e 100644 --- a/orchid-api/src/atom.rs +++ b/orchid-api/src/atom.rs @@ -120,6 +120,13 @@ impl Request for AtomPrint { type Response = String; } +#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)] +#[extends(ExtHostReq)] +pub struct ExtAtomPrint(pub Atom); +impl Request for ExtAtomPrint { + type Response = String; +} + /// Requests that apply to an existing atom instance #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)] #[extends(HostExtReq)] diff --git a/orchid-api/src/interner.rs b/orchid-api/src/interner.rs index da67472..1263d1d 100644 --- a/orchid-api/src/interner.rs +++ b/orchid-api/src/interner.rs @@ -1,5 +1,4 @@ use std::num::NonZeroU64; -use std::sync::Arc; use orchid_api_derive::{Coding, Hierarchy}; use orchid_api_traits::Request; diff --git a/orchid-api/src/parser.rs b/orchid-api/src/parser.rs index e74ce62..043d189 100644 --- a/orchid-api/src/parser.rs +++ b/orchid-api/src/parser.rs @@ -1,13 +1,20 @@ use std::num::NonZeroU64; -use orchid_api_derive::{Coding, Hierarchy}; +use orchid_api_derive::{Coding, Decode, Encode, Hierarchy}; use orchid_api_traits::Request; use crate::{Comment, HostExtReq, OrcResult, SysId, TokenTree}; -#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)] +#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)] pub struct ParsId(pub NonZeroU64); +// impl orchid_api_traits::Decode for ParsId { +// async fn decode(mut read: +// std::pin::Pin<&mut R>) -> Self { +// Self(orchid_api_traits::Decode::decode(read.as_mut()).await) +// } +// } + #[derive(Clone, Debug, Coding, Hierarchy)] #[extends(HostExtReq)] pub struct ParseLine { diff --git a/orchid-api/src/proto.rs b/orchid-api/src/proto.rs index e9716e8..4b3361a 100644 --- a/orchid-api/src/proto.rs +++ b/orchid-api/src/proto.rs @@ -22,8 +22,9 @@ //! be preserved. Toolkits must ensure that the client code is able to observe //! the ordering of messages. -use std::io::{Read, Write}; +use std::pin::Pin; +use async_std::io::{Read, Write}; use orchid_api_derive::{Coding, Hierarchy}; use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact}; @@ -34,15 +35,15 @@ pub struct HostHeader { pub log_strategy: logging::LogStrategy, } impl Decode for HostHeader { - fn decode(read: &mut R) -> Self { - read_exact(read, HOST_INTRO); - Self { log_strategy: logging::LogStrategy::decode(read) } + async fn decode(mut read: Pin<&mut R>) -> Self { + read_exact(read.as_mut(), HOST_INTRO).await; + Self { log_strategy: logging::LogStrategy::decode(read).await } } } impl Encode for HostHeader { - fn encode(&self, write: &mut W) { - write_exact(write, HOST_INTRO); - self.log_strategy.encode(write) + async fn encode(&self, mut write: Pin<&mut W>) { + write_exact(write.as_mut(), HOST_INTRO).await; + self.log_strategy.encode(write).await } } @@ -52,16 +53,16 @@ pub struct ExtensionHeader { pub systems: Vec, } impl Decode for ExtensionHeader { - fn decode(read: &mut R) -> Self { - read_exact(read, EXT_INTRO); - Self { name: String::decode(read), systems: Vec::decode(read) } + async fn decode(mut read: Pin<&mut R>) -> Self { + read_exact(read.as_mut(), EXT_INTRO).await; + Self { name: String::decode(read.as_mut()).await, systems: Vec::decode(read).await } } } impl Encode for ExtensionHeader { - fn encode(&self, write: &mut W) { - write_exact(write, EXT_INTRO); - self.name.encode(write); - self.systems.encode(write) + async fn encode(&self, mut write: Pin<&mut W>) { + write_exact(write.as_mut(), EXT_INTRO).await; + self.name.encode(write.as_mut()).await; + self.systems.encode(write).await } } @@ -78,6 +79,7 @@ pub enum ExtHostReq { Ping(Ping), IntReq(interner::IntReq), Fwd(atom::Fwd), + ExtAtomPrint(atom::ExtAtomPrint), SysFwd(system::SysFwd), ExprReq(expr::ExprReq), SubLex(lexer::SubLex), @@ -150,32 +152,37 @@ impl MsgSet for HostMsgSet { mod tests { use orchid_api_traits::enc_vec; use ordered_float::NotNan; + use test_executors::spin_on; use super::*; #[test] fn host_header_enc() { - let hh = HostHeader { log_strategy: logging::LogStrategy::File("SomeFile".to_string()) }; - let mut enc = &enc_vec(&hh)[..]; - eprintln!("Encoded to {enc:?}"); - HostHeader::decode(&mut enc); - assert_eq!(enc, []); + spin_on(async { + let hh = HostHeader { log_strategy: logging::LogStrategy::File("SomeFile".to_string()) }; + let mut enc = &enc_vec(&hh).await[..]; + eprintln!("Encoded to {enc:?}"); + HostHeader::decode(Pin::new(&mut enc)).await; + assert_eq!(enc, []); + }) } #[test] fn ext_header_enc() { - let eh = ExtensionHeader { - name: "my_extension".to_string(), - systems: vec![system::SystemDecl { - id: system::SysDeclId(1.try_into().unwrap()), - name: "misc".to_string(), - depends: vec!["std".to_string()], - priority: NotNan::new(1f64).unwrap(), - }], - }; - let mut enc = &enc_vec(&eh)[..]; - eprintln!("Encoded to {enc:?}"); - ExtensionHeader::decode(&mut enc); - assert_eq!(enc, []) + spin_on(async { + let eh = ExtensionHeader { + name: "my_extension".to_string(), + systems: vec![system::SystemDecl { + id: system::SysDeclId(1.try_into().unwrap()), + name: "misc".to_string(), + depends: vec!["std".to_string()], + priority: NotNan::new(1f64).unwrap(), + }], + }; + let mut enc = &enc_vec(&eh).await[..]; + eprintln!("Encoded to {enc:?}"); + ExtensionHeader::decode(Pin::new(&mut enc)).await; + assert_eq!(enc, []) + }) } } diff --git a/orchid-base/Cargo.toml b/orchid-base/Cargo.toml index 2edff13..9bce227 100644 --- a/orchid-base/Cargo.toml +++ b/orchid-base/Cargo.toml @@ -20,6 +20,7 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" } orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" } orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" } ordered-float = "4.6.0" +regex = "1.11.1" rust-embed = "8.5.0" rust_decimal = "1.36.0" some_executor = "0.4.0" diff --git a/orchid-base/src/builtin.rs b/orchid-base/src/builtin.rs index b86725d..5f2116f 100644 --- a/orchid-base/src/builtin.rs +++ b/orchid-base/src/builtin.rs @@ -12,9 +12,9 @@ use crate::api; /// There are no ordering guarantees about these pub trait ExtPort { fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>; - fn recv<'a, 's: 'a>( - &'s self, - cb: Box LocalBoxFuture<'a, ()> + 'a>, + fn recv<'a>( + &'a self, + cb: Box LocalBoxFuture<'_, ()> + 'a>, ) -> LocalBoxFuture<'a, ()>; } @@ -26,7 +26,7 @@ impl ExtInit { pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await } pub async fn recv<'a, 's: 'a>( &'s self, - cb: Box LocalBoxFuture<'a, ()> + 'a>, + cb: Box LocalBoxFuture<'_, ()> + 'a>, ) { self.port.recv(Box::new(cb)).await } diff --git a/orchid-base/src/format.rs b/orchid-base/src/format.rs new file mode 100644 index 0000000..af9ee0b --- /dev/null +++ b/orchid-base/src/format.rs @@ -0,0 +1,89 @@ +use std::convert::Infallible; +use std::iter; +use std::rc::Rc; +use std::str::FromStr; + +use itertools::Itertools; +use regex::Regex; + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct FmtUnit { + pub subs: Vec, + pub variants: Rc, +} +impl FmtUnit { + pub fn new(variants: Rc, subs: impl IntoIterator) -> Self { + Self { subs: subs.into_iter().collect(), variants } + } +} +impl From for FmtUnit +where Variants: From +{ + fn from(value: T) -> Self { Self { subs: vec![], variants: Rc::new(Variants::from(value)) } } +} +impl FromStr for FmtUnit { + type Err = Infallible; + fn from_str(s: &str) -> Result { + Ok(Self { subs: vec![], variants: Rc::new(Variants::new([s])) }) + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub enum FmtElement { + Sub(u8), + String(Rc), + InlineSub(u8), +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct Variants(pub Vec>); +impl Variants { + pub fn new<'a>(variants: impl IntoIterator) -> Self { + let re = Regex::new(r"(?\{\d+?-?\})|(\{\{)|(\}\})").unwrap(); + Self(Vec::from_iter(variants.into_iter().map(|s: &str| { + let matches = re.captures_iter(s); + let slots = matches.into_iter().filter_map(|m| m.name("tpl")).map(|tpl| { + let no_opencurly = tpl.as_str().strip_prefix("{").expect("required by regex"); + let maybe_dash = no_opencurly.strip_suffix("}").expect("required by regex"); + let (num, had_dash) = + maybe_dash.strip_suffix('-').map_or((maybe_dash, false), |s| (s, true)); + let idx = num.parse::().expect("Decimal digits required by regex"); + (tpl.range(), idx, had_dash) + }); + (iter::once(None).chain(slots.into_iter().map(Some)).chain(None).tuple_windows()) + .flat_map(|(l, r)| { + let string = match (l, &r) { + (None, Some((r, ..))) => &s[..r.start], + (Some((r1, ..)), Some((r2, ..))) => &s[r1.end..r2.start], + (Some((r, ..)), None) => &s[r.end..], + (None, None) => s, + }; + let str_item = FmtElement::String(Rc::new(string.to_string())); + match r { + None => itertools::Either::Left([str_item]), + Some((_, idx, inline)) => itertools::Either::Right([str_item, match inline { + true => FmtElement::InlineSub(idx), + false => FmtElement::Sub(idx), + }]), + } + .into_iter() + }) + .coalesce(|left, right| match (left, right) { + (FmtElement::String(left), FmtElement::String(right)) => + Ok(FmtElement::String(Rc::new(left.to_string() + right.as_str()))), + tuple => Err(tuple), + }) + .collect_vec() + }))) + } +} +impl From for Variants { + fn from(value: String) -> Self { Self(vec![vec![FmtElement::String(Rc::new(value))]]) } +} +impl From> for Variants { + fn from(value: Rc) -> Self { Self(vec![vec![FmtElement::String(value)]]) } +} +impl FromStr for Variants { + type Err = Infallible; + fn from_str(s: &str) -> Result { Ok(Self::new([s])) } +} diff --git a/orchid-base/src/interner.rs b/orchid-base/src/interner.rs index 4ba6ee0..8d1995d 100644 --- a/orchid-base/src/interner.rs +++ b/orchid-base/src/interner.rs @@ -247,7 +247,7 @@ impl Interner { tok } /// Extern an identifier; query the data it represents if not known locally - async fn ex(&self, marker: M) -> Tok { + pub async fn ex(&self, marker: M) -> Tok { if let Some(tok) = M::Interned::bimap(&mut *self.interners.lock().await).by_marker(marker) { return tok; } @@ -284,6 +284,7 @@ pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) { #[cfg(test)] mod test { use std::num::NonZero; + use std::pin::Pin; use orchid_api_traits::{Decode, enc_vec}; use test_executors::spin_on; @@ -300,9 +301,11 @@ mod test { #[test] fn test_coding() { - let coded = api::TStr(NonZero::new(3u64).unwrap()); - let mut enc = &enc_vec(&coded)[..]; - api::TStr::decode(&mut enc); - assert_eq!(enc, [], "Did not consume all of {enc:?}") + spin_on(async { + let coded = api::TStr(NonZero::new(3u64).unwrap()); + let mut enc = &enc_vec(&coded).await[..]; + api::TStr::decode(Pin::new(&mut enc)).await; + assert_eq!(enc, [], "Did not consume all of {enc:?}") + }) } } diff --git a/orchid-base/src/lib.rs b/orchid-base/src/lib.rs index de4fb51..7307df1 100644 --- a/orchid-base/src/lib.rs +++ b/orchid-base/src/lib.rs @@ -9,6 +9,7 @@ pub mod clone; pub mod combine; pub mod error; pub mod event; +pub mod format; pub mod id_store; pub mod interner; pub mod join; diff --git a/orchid-base/src/macros.rs b/orchid-base/src/macros.rs index ad8b576..fd6f521 100644 --- a/orchid-base/src/macros.rs +++ b/orchid-base/src/macros.rs @@ -1,4 +1,5 @@ use std::marker::PhantomData; +use std::rc::Rc; use std::sync::Arc; use async_std::stream; @@ -22,13 +23,14 @@ impl MacroSlot<'_> { trait_set! { pub trait MacroAtomToApi = for<'a> FnMut(&'a A) -> LocalBoxFuture<'a, api::MacroToken>; - pub trait MacroAtomFromApi<'a, A> = FnMut(&api::Atom) -> MTok<'a, A>; + pub trait MacroAtomFromApi<'a, A> = + for<'b> FnMut(&'b api::Atom) -> LocalBoxFuture<'b, MTok<'a, A>>; } #[derive(Clone, Debug)] pub struct MTree<'a, A> { pub pos: Pos, - pub tok: Arc>, + pub tok: Rc>, } impl<'a, A> MTree<'a, A> { pub(crate) async fn from_api( @@ -38,7 +40,7 @@ impl<'a, A> MTree<'a, A> { ) -> Self { Self { pos: Pos::from_api(&api.location, i).await, - tok: Arc::new(MTok::from_api(&api.token, do_atom, i).await), + tok: Rc::new(MTok::from_api(&api.token, i, do_atom).await), } } pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi) -> api::MacroTree { @@ -66,17 +68,17 @@ pub enum MTok<'a, A> { impl<'a, A> MTok<'a, A> { pub(crate) async fn from_api( api: &api::MacroToken, - do_atom: &mut impl MacroAtomFromApi<'a, A>, i: &Interner, + do_atom: &mut impl MacroAtomFromApi<'a, A>, ) -> Self { match_mapping!(&api, api::MacroToken => MTok::<'a, A> { - Lambda(x => mtreev_from_api(x, do_atom, i).await, b => mtreev_from_api(b, do_atom, i).await), + Lambda(x => mtreev_from_api(x, i, do_atom).await, b => mtreev_from_api(b, i, do_atom).await), Name(t => Sym::from_api(*t, i).await), Slot(tk => MacroSlot(*tk, PhantomData)), - S(p.clone(), b => mtreev_from_api(b, do_atom, i).await), + S(p.clone(), b => mtreev_from_api(b, i, do_atom).await), Ph(ph => Ph::from_api(ph, i).await), } { - api::MacroToken::Atom(a) => do_atom(a) + api::MacroToken::Atom(a) => do_atom(a).await }) } pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi) -> api::MacroToken { @@ -93,13 +95,13 @@ impl<'a, A> MTok<'a, A> { MTok::Atom(a) => do_atom(a).await, }) } - pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Arc::new(self) } } + pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Rc::new(self) } } } pub async fn mtreev_from_api<'a, 'b, A>( api: impl IntoIterator, - do_atom: &mut impl MacroAtomFromApi<'a, A>, i: &Interner, + do_atom: &mut impl MacroAtomFromApi<'a, A>, ) -> Vec> { let do_atom_lk = Mutex::new(do_atom); stream::from_iter(api) diff --git a/orchid-base/src/msg.rs b/orchid-base/src/msg.rs index ea243f9..80fda1d 100644 --- a/orchid-base/src/msg.rs +++ b/orchid-base/src/msg.rs @@ -6,7 +6,7 @@ use orchid_api_traits::{Decode, Encode}; pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result<()> { let mut len_buf = vec![]; - u32::try_from(msg.len()).unwrap().encode(&mut len_buf); + u32::try_from(msg.len()).unwrap().encode(Pin::new(&mut len_buf)).await; write.write_all(&len_buf).await?; write.write_all(msg).await?; write.flush().await @@ -15,7 +15,7 @@ pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result pub async fn recv_msg(mut read: Pin<&mut impl Read>) -> io::Result> { let mut len_buf = [0u8; (u32::BITS / 8) as usize]; read.read_exact(&mut len_buf).await?; - let len = u32::decode(&mut &len_buf[..]); + let len = u32::decode(Pin::new(&mut &len_buf[..])).await; let mut msg = vec![0u8; len as usize]; read.read_exact(&mut msg).await?; Ok(msg) diff --git a/orchid-base/src/parse.rs b/orchid-base/src/parse.rs index 8104424..9427dbc 100644 --- a/orchid-base/src/parse.rs +++ b/orchid-base/src/parse.rs @@ -33,6 +33,7 @@ impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> { pub async fn i(&self, arg: &(impl Internable + ?Sized)) -> Tok { self.interner.i(arg).await } + pub fn interner(&self) -> &'a Interner { self.interner } pub fn split_at(self, pos: u32) -> (Self, Self) { let Self { prev, cur, interner } = self; let fst = Self { prev, cur: &cur[..pos as usize], interner }; @@ -177,7 +178,7 @@ pub async fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>( Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }), t => Err(mk_errv( snip.i("Expected specific keyword").await, - format!("Expected {tok} but found {t}"), + format!("Expected {tok} but found {:?}", t.print().await), [Pos::Range(head.range.clone()).into()], )), } @@ -280,7 +281,7 @@ pub async fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>( t => { return Err(mk_errv( tail.i("Unrecognized name end").await, - format!("Names cannot end with {t} tokens"), + format!("Names cannot end with {:?} tokens", t.print().await), [Pos::Range(name.range.clone()).into()], )); }, diff --git a/orchid-base/src/reqnot.rs b/orchid-base/src/reqnot.rs index 03309ca..4f26985 100644 --- a/orchid-base/src/reqnot.rs +++ b/orchid-base/src/reqnot.rs @@ -4,6 +4,7 @@ use std::future::Future; use std::marker::PhantomData; use std::mem; use std::ops::{BitAnd, Deref}; +use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -74,7 +75,7 @@ impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> { pub async fn respond(&self, response: &impl Encode) -> Receipt<'a> { assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id); let mut buf = (!self.id).to_be_bytes().to_vec(); - response.encode(&mut buf); + response.encode(Pin::new(&mut buf)).await; let mut send = clone_box(&*self.reqnot().0.lock().await.send); (send)(&buf, self.parent.clone()).await; Receipt(PhantomData) @@ -126,18 +127,19 @@ impl ReqNot { let mut g = self.0.lock().await; let (id, payload) = get_id(message); if id == 0 { - let mut notif = clone_box(&*g.notif); + let mut notif_cb = clone_box(&*g.notif); mem::drop(g); - notif(::Notif::decode(&mut &payload[..]), self.clone()).await + let notif_val = ::Notif::decode(Pin::new(&mut &payload[..])).await; + notif_cb(notif_val, self.clone()).await } else if 0 < id.bitand(1 << 63) { let sender = g.responses.remove(&!id).expect("Received response for invalid message"); sender.send(message.to_vec()).await.unwrap(); } else { - let message = ::Req::decode(&mut &payload[..]); - let mut req = clone_box(&*g.req); + let message = ::Req::decode(Pin::new(&mut &payload[..])).await; + let mut req_cb = clone_box(&*g.req); mem::drop(g); let rn = self.clone(); - req(RequestHandle::new(rn, id), message).await; + req_cb(RequestHandle::new(rn, id), message).await; } } @@ -145,7 +147,7 @@ impl ReqNot { let mut send = clone_box(&*self.0.lock().await.send); let mut buf = vec![0; 8]; let msg: ::Notif = notif.into(); - msg.encode(&mut buf); + msg.encode(Pin::new(&mut buf)).await; send(&buf, self.clone()).await } } @@ -180,7 +182,7 @@ impl DynRequester for ReqNot { let id = g.id; g.id += 1; let mut buf = id.to_be_bytes().to_vec(); - req.encode(&mut buf); + req.encode(Pin::new(&mut buf)).await; let (send, recv) = channel::bounded(1); g.responses.insert(id, send); let mut send = clone_box(&*g.send); @@ -206,7 +208,7 @@ pub trait Requester: DynRequester { } impl Requester for This { async fn request>(&self, data: R) -> R::Response { - R::Response::decode(&mut &self.raw_request(data.into()).await[..]) + R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await } } diff --git a/orchid-base/src/tree.rs b/orchid-base/src/tree.rs index e293cb4..b69609c 100644 --- a/orchid-base/src/tree.rs +++ b/orchid-base/src/tree.rs @@ -1,6 +1,6 @@ use std::borrow::Borrow; -use std::cell::RefCell; use std::fmt::{self, Debug, Display}; +use std::future::{Future, ready}; use std::iter; use std::marker::PhantomData; use std::ops::Range; @@ -9,7 +9,7 @@ use std::sync::Arc; pub use api::PhKind; use async_std::stream; use async_std::sync::Mutex; -use futures::future::LocalBoxFuture; +use futures::future::{LocalBoxFuture, join_all}; use futures::{FutureExt, StreamExt}; use itertools::Itertools; use never::Never; @@ -47,15 +47,22 @@ pub fn recur<'a, A: AtomRepr, X: ExtraTok>( }) } -pub trait AtomRepr: fmt::Display + Clone + fmt::Debug { +pub trait AtomRepr: Clone { type Ctx: ?Sized; - fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self; - fn to_api(&self) -> orchid_api::Atom; + fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> impl Future; + fn to_api(&self) -> impl Future + '_; + fn print(&self) -> impl Future + '_; } impl AtomRepr for Never { type Ctx = Never; - fn from_api(_: &api::Atom, _: Pos, _: &mut Self::Ctx) -> Self { panic!() } - fn to_api(&self) -> orchid_api::Atom { match *self {} } + #[allow(unreachable_code)] + fn from_api(_: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> impl Future { + ready(match *ctx {}) + } + #[allow(unreachable_code)] + fn to_api(&self) -> impl Future + '_ { ready(match *self {}) } + #[allow(unreachable_code)] + fn print(&self) -> impl Future + '_ { ready(match *self {}) } } #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] @@ -79,7 +86,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> { pub async fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx, i: &Interner) -> Self { let tok = match_mapping!(&tt.token, api::Token => Token::<'b, A, X> { BR, NS, - Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx)), + Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx).await), Bottom(e => OrcErrv::from_api(e, i).await), LambdaHead(arg => ttv_from_api(arg, ctx, i).await), Name(n => Tok::from_api(*n, i).await), @@ -94,7 +101,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> { pub async fn to_api(&self, do_extra: &mut impl RefDoExtra) -> api::TokenTree { let token = match_mapping!(&self.tok, Token => api::Token { - Atom(a.to_api()), + Atom(a.to_api().await), BR, NS, Bottom(e.to_api()), @@ -111,20 +118,20 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> { api::TokenTree { range: self.range.clone(), token } } - pub fn into_api( + pub async fn into_api( self, do_extra: &mut impl FnMut(X, Range) -> api::TokenTree, ) -> api::TokenTree { let token = match self.tok { - Token::Atom(a) => api::Token::Atom(a.to_api()), + Token::Atom(a) => api::Token::Atom(a.to_api().await), Token::BR => api::Token::BR, Token::NS => api::Token::NS, Token::Bottom(e) => api::Token::Bottom(e.to_api()), Token::Comment(c) => api::Token::Comment(c.clone()), - Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra)), + Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra).await), Token::Name(n) => api::Token::Name(n.to_api()), Token::Slot(tt) => api::Token::Slot(tt.ticket()), - Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra)), + Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra).await), Token::Ph(Ph { kind, name }) => api::Token::Ph(api::Placeholder { name: name.to_api(), kind }), Token::X(x) => return do_extra(x, self.range.clone()), @@ -146,10 +153,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> { body.insert(0, Token::LambdaHead(arg).at(arg_range)); Token::S(Paren::Round, body).at(s_range) } -} - -impl Display for TokTree<'_, A, X> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) } + pub async fn print(&self) -> String { self.tok.print().await } } pub async fn ttv_from_api( @@ -161,7 +165,7 @@ pub async fn ttv_from_api( stream::from_iter(tokv.into_iter()) .then(|t| async { let t = t; - TokTree::::from_api(t.borrow(), *ctx_lk.lock().await, i).await + TokTree::::from_api(t.borrow(), *ctx_lk.lock().await, i).boxed_local().await }) .collect() .await @@ -178,11 +182,15 @@ pub async fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>( output } -pub fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>( +pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>( tokv: impl IntoIterator>, do_extra: &mut impl FnMut(X, Range) -> api::TokenTree, ) -> Vec { - tokv.into_iter().map(|t| t.into_api(do_extra)).collect_vec() + let mut new_tokv = Vec::new(); + for item in tokv { + new_tokv.push(item.into_api(do_extra).await) + } + new_tokv } /// This takes a position and not a range because it assigns the range to @@ -237,50 +245,32 @@ impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> { _ => None, } } -} -impl Display for Token<'_, A, X> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - thread_local! { - static PAREN_LEVEL: RefCell = 0.into(); - } - fn get_indent() -> usize { PAREN_LEVEL.with_borrow(|t| *t) } - fn with_indent(f: impl FnOnce() -> T) -> T { - PAREN_LEVEL.with_borrow_mut(|t| *t += 1); - let r = f(); - PAREN_LEVEL.with_borrow_mut(|t| *t -= 1); - r - } + pub async fn print(&self) -> String { match self { - Self::Atom(a) => f.write_str(&indent(&format!("{a} "), get_indent(), false)), - Self::BR => write!(f, "\n{}", " ".repeat(get_indent())), - Self::Bottom(err) if err.len() == 1 => write!(f, "Bottom({}) ", err.one().unwrap()), - Self::Bottom(err) => { - write!(f, "Botttom(\n{}) ", indent(&err.to_string(), get_indent() + 1, true)) - }, - Self::Comment(c) => write!(f, "--[{c}]-- "), - Self::LambdaHead(arg) => with_indent(|| write!(f, "\\ {} . ", ttv_fmt(arg))), - Self::NS => f.write_str(":: "), - Self::Name(n) => write!(f, "{n} "), - Self::Slot(th) => write!(f, "{th} "), + Self::Atom(a) => a.print().await, + Self::BR => "\n".to_string(), + Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()), + Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())), + Self::Comment(c) => format!("--[{c}]-- "), + Self::LambdaHead(arg) => format!("\\ {} . ", indent(&ttv_fmt(arg).await)), + Self::NS => ":: ".to_string(), + Self::Name(n) => format!("{n} "), + Self::Slot(th) => format!("{th} "), Self::Ph(Ph { kind, name }) => match &kind { - PhKind::Scalar => write!(f, "${name}"), + PhKind::Scalar => format!("${name}"), PhKind::Vector { at_least_one, priority } => { - if *at_least_one { - write!(f, ".")? - } - write!(f, "..${name}")?; - if 0 < *priority { write!(f, "{priority}") } else { Ok(()) } + let prefix = if *at_least_one { "..." } else { ".." }; + let suffix = if 0 < *priority { format!(":{priority}") } else { String::new() }; + format!("{prefix}${name}{suffix}") }, }, Self::S(p, b) => { let (lp, rp, _) = PARENS.iter().find(|(_, _, par)| par == p).unwrap(); - write!(f, "{lp} ")?; - with_indent(|| f.write_str(&ttv_fmt(b)))?; - write!(f, "{rp} ") + format!("{lp} {}{rp} ", indent(&ttv_fmt(b).await)) }, - Self::X(x) => write!(f, "{x} "), - Self::Macro(None) => write!(f, "macro "), - Self::Macro(Some(prio)) => write!(f, "macro({prio})"), + Self::X(x) => format!("{x} "), + Self::Macro(None) => "macro ".to_string(), + Self::Macro(Some(prio)) => format!("macro({prio})"), } } } @@ -290,21 +280,13 @@ pub fn ttv_range(ttv: &[TokTree<'_, impl AtomRepr, impl ExtraTok>]) -> Range( +pub async fn ttv_fmt<'a: 'b, 'b>( ttv: impl IntoIterator>, ) -> String { - ttv.into_iter().join("") + join_all(ttv.into_iter().map(|tt| tt.print())).await.join("") } -pub fn indent(s: &str, lvl: usize, first: bool) -> String { - if first { - s.replace("\n", &("\n".to_string() + &" ".repeat(lvl))) - } else if let Some((fst, rest)) = s.split_once('\n') { - fst.to_string() + "\n" + &indent(rest, lvl, true) - } else { - s.to_string() - } -} +pub fn indent(s: &str) -> String { s.replace("\n", "\n ") } #[derive(Clone, Debug)] pub struct Ph { diff --git a/orchid-extension/src/atom.rs b/orchid-extension/src/atom.rs index 1db82f8..3c39f55 100644 --- a/orchid-extension/src/atom.rs +++ b/orchid-extension/src/atom.rs @@ -1,12 +1,13 @@ use std::any::{Any, TypeId, type_name}; use std::fmt; use std::future::Future; -use std::io::{Read, Write}; use std::marker::PhantomData; use std::ops::Deref; +use std::pin::Pin; use std::rc::Rc; use ahash::HashMap; +use async_std::io::{Read, Write}; use async_std::stream; use dyn_clone::{DynClone, clone_box}; use futures::future::LocalBoxFuture; @@ -34,7 +35,7 @@ pub trait AtomCard: 'static + Sized { pub trait AtomicVariant {} pub trait Atomic: 'static + Sized { type Variant: AtomicVariant; - type Data: Clone + Coding + Sized; + type Data: Clone + Coding + Sized + 'static; fn reg_reqs() -> MethodSetBuilder; } impl AtomCard for A { @@ -102,10 +103,10 @@ impl ForeignAtom<'static> { let rep = (self.ctx.reqnot.request(api::Fwd( self.atom.clone(), Sym::parse(M::NAME, &self.ctx.i).await.unwrap().tok().to_api(), - enc_vec(&m), + enc_vec(&m).await, ))) .await?; - Some(M::Response::decode(&mut &rep[..])) + Some(M::Response::decode(Pin::new(&mut &rep[..])).await) } } impl fmt::Display for ForeignAtom<'_> { @@ -118,10 +119,13 @@ impl fmt::Debug for ForeignAtom<'_> { } impl AtomRepr for ForeignAtom<'_> { type Ctx = SysCtx; - fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self { + async fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self { Self { atom: atom.clone(), _life: PhantomData, ctx: ctx.clone(), expr: None, pos } } - fn to_api(&self) -> orchid_api::Atom { self.atom.clone() } + async fn to_api(&self) -> orchid_api::Atom { self.atom.clone() } + async fn print(&self) -> String { + self.ctx.reqnot.request(api::ExtAtomPrint(self.atom.clone())).await + } } pub struct NotTypAtom { @@ -151,8 +155,8 @@ trait_set! { trait AtomReqCb = for<'a> Fn( &'a A, SysCtx, - &'a mut dyn Read, - &'a mut dyn Write + Pin<&'a mut dyn Read>, + Pin<&'a mut dyn Write>, ) -> LocalBoxFuture<'a, ()> } @@ -167,8 +171,9 @@ impl MethodSetBuilder { assert!(!M::NAME.is_empty(), "AtomMethod::NAME cannoot be empty"); self.handlers.push(( M::NAME, - Rc::new(move |a: &A, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write| { - async { Supports::::handle(a, ctx, M::decode(req)).await.encode(rep) }.boxed_local() + Rc::new(move |a: &A, ctx: SysCtx, req: Pin<&mut dyn Read>, rep: Pin<&mut dyn Write>| { + async { Supports::::handle(a, ctx, M::decode(req).await).await.encode(rep).await } + .boxed_local() }), )); self @@ -197,8 +202,8 @@ impl MethodSet { atom: &'a A, ctx: SysCtx, key: Sym, - req: &'a mut dyn Read, - rep: &'a mut dyn Write, + req: Pin<&'a mut dyn Read>, + rep: Pin<&'a mut dyn Write>, ) -> bool { match self.handlers.get(&key) { None => false, @@ -228,14 +233,14 @@ impl TypAtom<'static, A> { expr, typ: Box::new(A::info()), }), - Ok(atm) => match downcast_atom::(atm) { + Ok(atm) => match downcast_atom::(atm).await { + Ok(tatom) => Ok(tatom), Err(fa) => Err(NotTypAtom { pos: fa.pos.clone(), ctx: fa.ctx.clone(), expr: fa.ex(), typ: Box::new(A::info()), }), - Ok(tatom) => Ok(tatom), }, } } @@ -243,15 +248,16 @@ impl TypAtom<'static, A> { impl TypAtom<'_, A> { pub async fn request(&self, req: M) -> M::Response where A: Supports { - M::Response::decode( + M::Response::decode(Pin::new( &mut &(self.data.ctx.reqnot.request(api::Fwd( self.data.atom.clone(), Sym::parse(M::NAME, &self.data.ctx.i).await.unwrap().tok().to_api(), - enc_vec(&req), + enc_vec(&req).await, ))) .await .unwrap()[..], - ) + )) + .await } } impl Deref for TypAtom<'_, A> { @@ -264,7 +270,7 @@ pub struct AtomCtx<'a>(pub &'a [u8], pub Option, pub SysCtx); pub trait AtomDynfo: 'static { fn tid(&self) -> TypeId; fn name(&self) -> &'static str; - fn decode(&self, ctx: AtomCtx<'_>) -> Box; + fn decode<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, Box>; fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>; fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>; fn print<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, String>; @@ -272,14 +278,14 @@ pub trait AtomDynfo: 'static { &'a self, ctx: AtomCtx<'a>, key: Sym, - req: &'b mut dyn Read, - rep: &'c mut dyn Write, + req: Pin<&'b mut dyn Read>, + rep: Pin<&'c mut dyn Write>, ) -> LocalBoxFuture<'a, bool>; fn command<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, OrcRes>>; fn serialize<'a, 'b: 'a>( &'a self, ctx: AtomCtx<'a>, - write: &'b mut dyn Write, + write: Pin<&'b mut dyn Write>, ) -> LocalBoxFuture<'a, Option>>; fn deserialize<'a>( &'a self, diff --git a/orchid-extension/src/atom_owned.rs b/orchid-extension/src/atom_owned.rs index 1143343..cc3da20 100644 --- a/orchid-extension/src/atom_owned.rs +++ b/orchid-extension/src/atom_owned.rs @@ -1,10 +1,11 @@ use std::any::{Any, TypeId, type_name}; use std::borrow::Cow; use std::future::Future; -use std::io::{Read, Write}; +use std::pin::Pin; use std::rc::Rc; use async_once_cell::OnceCell; +use async_std::io::{Read, Write}; use futures::FutureExt; use futures::future::{LocalBoxFuture, ready}; use itertools::Itertools; @@ -31,8 +32,8 @@ impl> AtomicFeaturesImpl(ctx.cted.inst().card()); - let mut data = enc_vec(&id); - rec.encode(&mut data).await; + let mut data = enc_vec(&id).await; + rec.encode(Pin::<&mut Vec>::new(&mut data)).await; api::Atom { drop: Some(api::AtomId(rec.id())), data, owner: ctx.id } }) } @@ -55,8 +56,11 @@ pub struct OwnedAtomDynfo { impl AtomDynfo for OwnedAtomDynfo { fn tid(&self) -> TypeId { TypeId::of::() } fn name(&self) -> &'static str { type_name::() } - fn decode(&self, AtomCtx(data, ..): AtomCtx) -> Box { - Box::new(::Data::decode(&mut &data[..])) + fn decode<'a>(&'a self, AtomCtx(data, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box> { + async { + Box::new(::Data::decode(Pin::new(&mut &data[..])).await) as Box + } + .boxed_local() } fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> { with_atom(id.unwrap(), &ctx, |a| a.remove()).dyn_call(ctx.clone(), arg) @@ -82,8 +86,8 @@ impl AtomDynfo for OwnedAtomDynfo { &'a self, AtomCtx(_, id, ctx): AtomCtx, key: Sym, - req: &'b mut dyn Read, - rep: &'c mut dyn Write, + req: Pin<&'b mut dyn Read>, + rep: Pin<&'c mut dyn Write>, ) -> LocalBoxFuture<'a, bool> { async move { with_atom(id.unwrap(), &ctx, |a| { @@ -110,11 +114,11 @@ impl AtomDynfo for OwnedAtomDynfo { fn serialize<'a, 'b: 'a>( &'a self, AtomCtx(_, id, ctx): AtomCtx<'a>, - write: &'b mut dyn Write, + mut write: Pin<&'b mut dyn Write>, ) -> LocalBoxFuture<'a, Option>> { async move { let id = id.unwrap(); - id.encode(write); + id.encode(write.as_mut()).await; with_atom(id, &ctx, |a| clone!(ctx; async move { a.dyn_serialize(ctx, write).await })) .await .map(|v| v.into_iter().map(|t| t.handle().tk).collect_vec()) @@ -138,20 +142,22 @@ impl AtomDynfo for OwnedAtomDynfo { } pub trait DeserializeCtx: Sized { - fn read(&mut self) -> T; + fn read(&mut self) -> impl Future; fn is_empty(&self) -> bool; fn assert_empty(self) { assert!(self.is_empty(), "Bytes found after decoding") } - fn decode(mut self) -> T { - let t = self.read(); - self.assert_empty(); - t + fn decode(mut self) -> impl Future { + async { + let t = self.read().await; + self.assert_empty(); + t + } } fn sys(&self) -> SysCtx; } struct DeserCtxImpl<'a>(&'a [u8], &'a SysCtx); impl DeserializeCtx for DeserCtxImpl<'_> { - fn read(&mut self) -> T { T::decode(&mut self.0) } + async fn read(&mut self) -> T { T::decode(Pin::new(&mut self.0)).await } fn is_empty(&self) -> bool { self.0.is_empty() } fn sys(&self) -> SysCtx { self.1.clone() } } @@ -228,7 +234,7 @@ pub trait OwnedAtom: Atomic + Any + Clone + 'static { fn serialize( &self, ctx: SysCtx, - write: &mut (impl Write + ?Sized), + write: Pin<&mut (impl Write + ?Sized)>, ) -> impl Future { assert_serializable::(); async { panic!("Either implement serialize or set Refs to Never for {}", type_name::()) } @@ -250,7 +256,7 @@ fn assert_serializable() { pub trait DynOwnedAtom: 'static { fn atom_tid(&self) -> TypeId; fn as_any_ref(&self) -> &dyn Any; - fn encode<'a>(&'a self, buffer: &'a mut dyn Write) -> LocalBoxFuture<'a, ()>; + fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()>; fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr>; fn dyn_call(self: Box, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'static, GExpr>; @@ -260,14 +266,14 @@ pub trait DynOwnedAtom: 'static { fn dyn_serialize<'a>( &'a self, ctx: SysCtx, - sink: &'a mut dyn Write, + sink: Pin<&'a mut dyn Write>, ) -> LocalBoxFuture<'a, Option>>; } impl DynOwnedAtom for T { fn atom_tid(&self) -> TypeId { TypeId::of::() } fn as_any_ref(&self) -> &dyn Any { self } - fn encode<'a>(&'a self, buffer: &'a mut dyn Write) -> LocalBoxFuture<'a, ()> { - async { self.val().await.as_ref().encode(buffer) }.boxed_local() + fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()> { + async { self.val().await.as_ref().encode(buffer).await }.boxed_local() } fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> { self.call_ref(ExprHandle::from_args(ctx, arg)).boxed_local() @@ -289,7 +295,7 @@ impl DynOwnedAtom for T { fn dyn_serialize<'a>( &'a self, ctx: SysCtx, - sink: &'a mut dyn Write, + sink: Pin<&'a mut dyn Write>, ) -> LocalBoxFuture<'a, Option>> { match TypeId::of::() == TypeId::of::<::Refs>() { true => ready(None).boxed_local(), diff --git a/orchid-extension/src/atom_thin.rs b/orchid-extension/src/atom_thin.rs index 1e07fa0..9fbd7ba 100644 --- a/orchid-extension/src/atom_thin.rs +++ b/orchid-extension/src/atom_thin.rs @@ -1,8 +1,9 @@ use std::any::{Any, TypeId, type_name}; use std::future::Future; -use std::io::Write; +use std::pin::Pin; use async_once_cell::OnceCell; +use async_std::io::{Read, Write}; use futures::FutureExt; use futures::future::LocalBoxFuture; use orchid_api_traits::{Coding, enc_vec}; @@ -24,8 +25,8 @@ impl> AtomicFeaturesImpl AtomFactory { AtomFactory::new(move |ctx| async move { let (id, _) = get_info::(ctx.cted.inst().card()); - let mut buf = enc_vec(&id); - self.encode(&mut buf); + let mut buf = enc_vec(&id).await; + self.encode(Pin::new(&mut buf)).await; api::Atom { drop: None, data: buf, owner: ctx.id } }) } @@ -39,53 +40,58 @@ pub struct ThinAtomDynfo { } impl AtomDynfo for ThinAtomDynfo { fn print<'a>(&self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, String> { - async move { T::decode(&mut &buf[..]).print(ctx).await }.boxed_local() + async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await }.boxed_local() } fn tid(&self) -> TypeId { TypeId::of::() } fn name(&self) -> &'static str { type_name::() } - fn decode(&self, AtomCtx(buf, ..): AtomCtx) -> Box { Box::new(T::decode(&mut &buf[..])) } + fn decode<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box> { + async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box }.boxed_local() + } fn call<'a>( &'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>, arg: api::ExprTicket, ) -> LocalBoxFuture<'a, GExpr> { - async move { T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg)).await } - .boxed_local() + Box::pin(async move { + T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await + }) } fn call_ref<'a>( &'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>, arg: api::ExprTicket, ) -> LocalBoxFuture<'a, GExpr> { - async move { T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg)).await } - .boxed_local() + Box::pin(async move { + T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await + }) } fn handle_req<'a, 'm1: 'a, 'm2: 'a>( &'a self, AtomCtx(buf, _, sys): AtomCtx<'a>, key: Sym, - req: &'m1 mut dyn std::io::Read, - rep: &'m2 mut dyn Write, + req: Pin<&'m1 mut dyn Read>, + rep: Pin<&'m2 mut dyn Write>, ) -> LocalBoxFuture<'a, bool> { - async move { + Box::pin(async move { let ms = self.ms.get_or_init(self.msbuild.pack(sys.clone())).await; - ms.dispatch(&T::decode(&mut &buf[..]), sys, key, req, rep).await - } - .boxed_local() + ms.dispatch(&T::decode(Pin::new(&mut &buf[..])).await, sys, key, req, rep).await + }) } fn command<'a>( &'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>, ) -> LocalBoxFuture<'a, OrcRes>> { - async move { T::decode(&mut &buf[..]).command(ctx).await }.boxed_local() + async move { T::decode(Pin::new(&mut &buf[..])).await.command(ctx).await }.boxed_local() } fn serialize<'a, 'b: 'a>( &'a self, ctx: AtomCtx<'a>, - write: &'b mut dyn Write, + write: Pin<&'b mut dyn Write>, ) -> LocalBoxFuture<'a, Option>> { - T::decode(&mut &ctx.0[..]).encode(write); - async { Some(Vec::new()) }.boxed_local() + Box::pin(async { + T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await; + Some(Vec::new()) + }) } fn deserialize<'a>( &'a self, @@ -94,11 +100,11 @@ impl AtomDynfo for ThinAtomDynfo { refs: &'a [api::ExprTicket], ) -> LocalBoxFuture<'a, api::Atom> { assert!(refs.is_empty(), "Refs found when deserializing thin atom"); - async { T::decode(&mut &data[..])._factory().build(ctx).await }.boxed_local() + async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await }.boxed_local() } fn drop<'a>(&'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> { async move { - let string_self = T::decode(&mut &buf[..]).print(ctx.clone()).await; + let string_self = T::decode(Pin::new(&mut &buf[..])).await.print(ctx.clone()).await; writeln!(ctx.logger, "Received drop signal for non-drop atom {string_self:?}"); } .boxed_local() diff --git a/orchid-extension/src/conv.rs b/orchid-extension/src/conv.rs index ff77de0..1fd14c3 100644 --- a/orchid-extension/src/conv.rs +++ b/orchid-extension/src/conv.rs @@ -35,7 +35,7 @@ impl TryFromExpr for TypAtom<'_, A> { async fn try_from_expr(expr: Expr) -> OrcRes { match expr.atom().await { Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &ex.ctx().i).await.into()), - Ok(f) => match downcast_atom(f) { + Ok(f) => match downcast_atom(f).await { Ok(a) => Ok(a), Err(f) => Err(err_type(f.pos(), &f.ctx().i).await.into()), }, diff --git a/orchid-extension/src/entrypoint.rs b/orchid-extension/src/entrypoint.rs index b0d58ce..7553975 100644 --- a/orchid-extension/src/entrypoint.rs +++ b/orchid-extension/src/entrypoint.rs @@ -3,6 +3,7 @@ use std::future::Future; use std::io::Write; use std::mem; use std::num::NonZero; +use std::pin::Pin; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -86,7 +87,7 @@ pub async fn with_atom_record<'a, F: Future, T>( let mut data = &atom.data[..]; let ctx = get_sys_ctx(atom.owner, reqnot).await; let inst = ctx.cted.inst(); - let id = api::AtomId::decode(&mut data); + let id = api::AtomId::decode(Pin::new(&mut data)).await; let atom_record = atom_by_idx(inst.card(), id).expect("Atom ID reserved"); cb(atom_record, ctx, id, data).await } @@ -114,23 +115,30 @@ impl ExtPort for ExtensionOwner { fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> { self.rn.receive(msg).boxed_local() } - fn recv<'a, 's: 'a>( - &'s self, - cb: Box LocalBoxFuture<'a, ()> + 'a>, + fn recv<'a>( + &'a self, + cb: Box LocalBoxFuture<'_, ()> + 'a>, ) -> LocalBoxFuture<'a, ()> { - async { cb(&self.out_recv.recv().await.unwrap()[..]).await }.boxed_local() + async { + let msg = self.out_recv.recv().await.unwrap(); + cb(&msg[..]).await + } + .boxed_local() } } async fn extension_main_logic(data: ExtensionData, spawner: Rc) { - let api::HostHeader { log_strategy } = api::HostHeader::decode(&mut std::io::stdin().lock()); + let api::HostHeader { log_strategy } = + api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await; let mut buf = Vec::new(); let decls = (data.systems.iter().enumerate()) .map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys)) .map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap()))) .collect_vec(); let systems = Rc::new(Mutex::new(HashMap::::new())); - api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() }.encode(&mut buf); + api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() } + .encode(Pin::new(&mut buf)) + .await; std::io::stdout().write_all(&buf).unwrap(); std::io::stdout().flush().unwrap(); let exiting = Arc::new(AtomicBool::new(false)); @@ -333,8 +341,8 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) let actx = AtomCtx(buf, atom.drop, ctx.clone()); match &atom_req { api::AtomReq::SerializeAtom(ser) => { - let mut buf = enc_vec(&id); - let refs_opt = nfo.serialize(actx, &mut buf).await; + let mut buf = enc_vec(&id).await; + let refs_opt = nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await; hand.handle(ser, &refs_opt.map(|refs| (buf, refs))).await }, api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) => @@ -343,7 +351,14 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) let api::Fwded(_, key, payload) = &fwded; let mut reply = Vec::new(); let key = Sym::from_api(*key, &i).await; - let some = nfo.handle_req(actx, key, &mut &payload[..], &mut reply).await; + let some = nfo + .handle_req( + actx, + key, + Pin::<&mut &[u8]>::new(&mut &payload[..]), + Pin::<&mut Vec<_>>::new(&mut reply), + ) + .await; hand.handle(fwded, &some.then_some(reply)).await }, api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => { @@ -374,7 +389,7 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) let api::DeserAtom(sys, buf, refs) = &deser; let mut read = &mut &buf[..]; let ctx = mk_ctx(*sys, hand.reqnot()).await; - let id = api::AtomId::decode(&mut read); + let id = api::AtomId::decode(Pin::new(&mut read)).await; let inst = ctx.cted.inst(); let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID"); hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, refs).await).await @@ -388,7 +403,7 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) for (k, v) in params { ctx.args.insert( Tok::from_api(k, &i).await, - mtreev_from_api(&v, &mut |_| panic!("No atom in macro prompt!"), &i).await, + mtreev_from_api(&v, &i, &mut |_| panic!("No atom in macro prompt!")).await, ); } let err_cascade = err_cascade(&i).await; diff --git a/orchid-extension/src/expr.rs b/orchid-extension/src/expr.rs index b249adb..6205785 100644 --- a/orchid-extension/src/expr.rs +++ b/orchid-extension/src/expr.rs @@ -21,22 +21,16 @@ pub struct ExprHandle { impl ExprHandle { pub(crate) fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } } pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() } + pub async fn clone(&self) -> Self { + self.ctx.reqnot.notify(api::Acquire(self.ctx.id, self.tk)).await; + Self { ctx: self.ctx.clone(), tk: self.tk } + } } impl fmt::Debug for ExprHandle { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ExprHandle({})", self.tk.0) } } -impl Clone for ExprHandle { - fn clone(&self) -> Self { - let SysCtx { reqnot, spawner, .. } = self.ctx.clone(); - let notif = api::Acquire(self.ctx.id, self.tk); - if let Err(e) = spawner.spawn_local(async move { reqnot.notify(notif).await }) { - panic!("Failed to schedule cloning notification, resource may not exist: {e}"); - } - Self { ctx: self.ctx.clone(), tk: self.tk } - } -} impl Drop for ExprHandle { fn drop(&mut self) { let notif = api::Release(self.ctx.id, self.tk); diff --git a/orchid-extension/src/func_atom.rs b/orchid-extension/src/func_atom.rs index fcf9049..0e8a0cf 100644 --- a/orchid-extension/src/func_atom.rs +++ b/orchid-extension/src/func_atom.rs @@ -1,9 +1,10 @@ use std::borrow::Cow; use std::collections::HashMap; use std::future::Future; -use std::io; +use std::pin::Pin; use std::rc::Rc; +use async_std::io::Write; use async_std::sync::Mutex; use futures::FutureExt; use futures::future::LocalBoxFuture; @@ -79,13 +80,13 @@ impl OwnedAtom for Fun { } } async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await } - async fn serialize(&self, _: SysCtx, sink: &mut (impl io::Write + ?Sized)) -> Self::Refs { - self.path.to_api().encode(sink); + async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs { + self.path.to_api().encode(write).await; self.args.clone() } async fn deserialize(ctx: impl DeserializeCtx, args: Self::Refs) -> Self { let sys = ctx.sys(); - let path = Sym::from_api(ctx.decode(), &sys.i).await; + let path = Sym::from_api(ctx.decode().await, &sys.i).await; let (arity, fun) = FUNS.with(|f| f.clone()).lock().await.get(&path).unwrap().clone(); Self { args, arity, path, fun } } diff --git a/orchid-extension/src/macros.rs b/orchid-extension/src/macros.rs index a791830..bb507a2 100644 --- a/orchid-extension/src/macros.rs +++ b/orchid-extension/src/macros.rs @@ -48,7 +48,7 @@ impl<'a> RuleCtx<'a> { return Err(err_cascade(&self.sys.i).await.into()); }; static ATOM_MSG: &str = "Returned atom from Rule recursion"; - Ok(mtreev_from_api(&treev, &mut |_| panic!("{ATOM_MSG}"), &self.sys.i).await) + Ok(mtreev_from_api(&treev, &self.sys.i, &mut |_| panic!("{ATOM_MSG}")).await) } pub fn getv(&mut self, key: &Tok) -> Vec> { self.args.remove(key).expect("Key not found") diff --git a/orchid-extension/src/system.rs b/orchid-extension/src/system.rs index 2c5b514..b183cf0 100644 --- a/orchid-extension/src/system.rs +++ b/orchid-extension/src/system.rs @@ -2,10 +2,10 @@ use core::fmt; use std::any::TypeId; use std::future::Future; use std::num::NonZero; +use std::pin::Pin; use std::rc::Rc; use std::sync::Arc; -use futures::FutureExt; use futures::future::LocalBoxFuture; use futures::task::LocalSpawn; use hashbrown::HashMap; @@ -68,8 +68,11 @@ pub fn atom_by_idx( } } -pub fn resolv_atom(sys: &(impl DynSystemCard + ?Sized), atom: &api::Atom) -> Box { - let tid = api::AtomId::decode(&mut &atom.data[..8]); +pub async fn resolv_atom( + sys: &(impl DynSystemCard + ?Sized), + atom: &api::Atom, +) -> Box { + let tid = api::AtomId::decode(Pin::new(&mut &atom.data[..8])).await; atom_by_idx(sys, tid).expect("Value of nonexistent type found") } @@ -102,21 +105,25 @@ impl DynSystem for T { fn dyn_lexers(&self) -> Vec { Self::lexers() } fn dyn_parsers(&self) -> Vec { Self::parsers() } fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec) -> LocalBoxFuture<'a, Receipt<'a>> { - Self::request(hand, ::Req::decode(&mut &req[..])).boxed_local() + Box::pin(async move { + Self::request(hand, ::Req::decode(Pin::new(&mut &req[..])).await).await + }) } fn card(&self) -> &dyn DynSystemCard { self } } -pub fn downcast_atom(foreign: ForeignAtom) -> Result, ForeignAtom> { +pub async fn downcast_atom(foreign: ForeignAtom<'_>) -> Result, ForeignAtom<'_>> +where A: AtomicFeatures { let mut data = &foreign.atom.data[..]; let ctx = foreign.ctx.clone(); + let value = api::AtomId::decode(Pin::new(&mut data)).await; let info_ent = (ctx.cted.deps().find(|s| s.id() == foreign.atom.owner)) .map(|sys| get_info::(sys.get_card())) - .filter(|(pos, _)| api::AtomId::decode(&mut data) == *pos); + .filter(|(pos, _)| value == *pos); match info_ent { None => Err(foreign), Some((_, info)) => { - let val = info.decode(AtomCtx(data, foreign.atom.drop, ctx)); + let val = info.decode(AtomCtx(data, foreign.atom.drop, ctx)).await; let value = *val.downcast::().expect("atom decode returned wrong type"); Ok(TypAtom { value, data: foreign }) }, diff --git a/orchid-host/Cargo.toml b/orchid-host/Cargo.toml index 5365ec7..9e26f66 100644 --- a/orchid-host/Cargo.toml +++ b/orchid-host/Cargo.toml @@ -6,7 +6,9 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-process = "2.3.0" async-std = "1.13.0" +async-stream = "0.3.6" derive_destructure = "1.0.0" futures = "0.3.31" hashbrown = "0.15.2" @@ -20,4 +22,5 @@ orchid-base = { version = "0.1.0", path = "../orchid-base" } ordered-float = "4.6.0" paste = "1.0.15" substack = "1.1.1" +test_executors = "0.3.2" trait-set = "0.3.0" diff --git a/orchid-host/src/atom.rs b/orchid-host/src/atom.rs new file mode 100644 index 0000000..cfbec68 --- /dev/null +++ b/orchid-host/src/atom.rs @@ -0,0 +1,98 @@ +use std::fmt; +use std::rc::{Rc, Weak}; + +use derive_destructure::destructure; +use orchid_base::location::Pos; +use orchid_base::reqnot::Requester; +use orchid_base::tree::AtomRepr; + +use crate::api; +use crate::ctx::Ctx; +use crate::expr::Expr; +use crate::system::System; + +#[derive(destructure)] +pub struct AtomData { + owner: System, + drop: Option, + data: Vec, +} +impl AtomData { + fn api(self) -> api::Atom { + let (owner, drop, data) = self.destructure(); + api::Atom { data, drop, owner: owner.id() } + } + fn api_ref(&self) -> api::Atom { + api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() } + } +} +impl Drop for AtomData { + fn drop(&mut self) { + if let Some(id) = self.drop { + self.owner.drop_atom(id); + } + } +} +impl fmt::Debug for AtomData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AtomData") + .field("drop", &self.drop) + .field("data", &self.data) + .field("owner", &self.owner.id()) + .finish() + } +} + +#[derive(Clone, Debug)] +pub struct AtomHand(Rc); +impl AtomHand { + pub(crate) async fn new(api::Atom { data, drop, owner }: api::Atom, ctx: &Ctx) -> Self { + let create = || async { + let owner = ctx.system_inst(owner).await.expect("Dropped system created atom"); + AtomHand(Rc::new(AtomData { data, owner, drop })) + }; + if let Some(id) = drop { + let mut owned_g = ctx.owned_atoms.write().await; + if let Some(data) = owned_g.get(&id) { + if let Some(atom) = data.upgrade() { + return atom; + } + } + let new = create().await; + owned_g.insert(id, new.downgrade()); + new + } else { + create().await + } + } + pub async fn call(self, arg: Expr) -> api::Expression { + let owner_sys = self.0.owner.clone(); + let reqnot = owner_sys.reqnot(); + owner_sys.ext().exprs().give_expr(arg.clone()); + match Rc::try_unwrap(self.0) { + Ok(data) => reqnot.request(api::FinalCall(data.api(), arg.id())).await, + Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await, + } + } + pub async fn req(&self, key: api::TStrv, req: Vec) -> Option> { + self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await + } + pub fn api_ref(&self) -> api::Atom { self.0.api_ref() } + pub async fn to_string(&self) -> String { + self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await + } + pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) } +} +impl AtomRepr for AtomHand { + type Ctx = Ctx; + async fn from_api(atom: &orchid_api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self { + Self::new(atom.clone(), &ctx).await + } + async fn to_api(&self) -> orchid_api::Atom { self.api_ref() } + async fn print(&self) -> String { self.to_string().await } +} + +pub struct WeakAtomHand(Weak); +impl WeakAtomHand { + pub fn upgrade(&self) -> Option { self.0.upgrade().map(AtomHand) } +} diff --git a/orchid-host/src/child.rs b/orchid-host/src/child.rs deleted file mode 100644 index 953117d..0000000 --- a/orchid-host/src/child.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::sync::Mutex; -use std::{fmt, io, mem, process}; - -use orchid_base::msg::{recv_msg, send_msg}; - -pub struct SharedChild { - child: process::Child, - stdin: Mutex, - stdout: Mutex, - debug: Option<(String, Mutex>)>, -} -impl SharedChild { - pub fn new( - command: &mut process::Command, - debug: Option<(&str, impl fmt::Write + 'static)>, - ) -> io::Result { - let mut child = - command.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?; - let stdin = Mutex::new(child.stdin.take().expect("Piped stdin above")); - let stdout = Mutex::new(child.stdout.take().expect("Piped stdout above")); - let debug = debug.map(|(n, w)| (n.to_string(), Mutex::new(Box::new(w) as Box))); - Ok(Self { child, stdin, stdout, debug }) - } - - pub fn send_msg(&self, msg: &[u8]) -> io::Result<()> { - if let Some((n, dbg)) = &self.debug { - let mut dbg = dbg.lock().unwrap(); - writeln!(dbg, "To {n}: {msg:?}").unwrap(); - } - send_msg(&mut *self.stdin.lock().unwrap(), msg) - } - - pub fn recv_msg(&self) -> io::Result> { - let msg = recv_msg(&mut *self.stdout.lock().unwrap()); - if let Some((n, dbg)) = &self.debug { - let mut dbg = dbg.lock().unwrap(); - writeln!(dbg, "From {n}: {msg:?}").unwrap(); - } - msg - } -} -impl Drop for SharedChild { - fn drop(&mut self) { mem::drop(self.child.kill()) } -} diff --git a/orchid-host/src/ctx.rs b/orchid-host/src/ctx.rs new file mode 100644 index 0000000..c9e697d --- /dev/null +++ b/orchid-host/src/ctx.rs @@ -0,0 +1,46 @@ +use std::cell::RefCell; +use std::num::NonZeroU16; +use std::rc::Rc; +use std::{fmt, ops}; + +use async_std::sync::RwLock; +use futures::task::LocalSpawn; +use hashbrown::HashMap; +use orchid_api::SysId; +use orchid_base::interner::Interner; + +use crate::api; +use crate::atom::WeakAtomHand; +use crate::system::{System, WeakSystem}; + +pub struct CtxData { + pub i: Rc, + pub spawn: Rc, + pub systems: RwLock>, + pub system_id: RefCell, + pub owned_atoms: RwLock>, +} +#[derive(Clone)] +pub struct Ctx(Rc); +impl ops::Deref for Ctx { + type Target = CtxData; + fn deref(&self) -> &Self::Target { &*self.0 } +} +impl Ctx { + pub(crate) async fn system_inst(&self, id: api::SysId) -> Option { + self.systems.read().await.get(&id).and_then(WeakSystem::upgrade) + } + pub(crate) fn next_sys_id(&self) -> api::SysId { + let mut g = self.system_id.borrow_mut(); + *g = g.checked_add(1).unwrap_or(NonZeroU16::new(1).unwrap()); + SysId(*g) + } +} +impl fmt::Debug for Ctx { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Ctx") + .field("i", &self.i) + .field("system_id", &self.system_id) + .finish_non_exhaustive() + } +} diff --git a/orchid-host/src/expr.rs b/orchid-host/src/expr.rs index a7ebe07..db3705a 100644 --- a/orchid-host/src/expr.rs +++ b/orchid-host/src/expr.rs @@ -1,10 +1,10 @@ use std::collections::VecDeque; use std::num::NonZeroU64; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, RwLock}; +use std::rc::{Rc, Weak}; +use std::sync::atomic::AtomicBool; -use hashbrown::HashMap; -use lazy_static::lazy_static; +use async_std::sync::RwLock; +use futures::FutureExt; use orchid_base::error::OrcErrv; use orchid_base::location::Pos; use orchid_base::match_mapping; @@ -12,69 +12,56 @@ use orchid_base::name::Sym; use orchid_base::tree::AtomRepr; use crate::api; -use crate::extension::AtomHand; +use crate::atom::AtomHand; +use crate::extension::Extension; -pub type ExprParseCtx = (); +pub type ExprParseCtx = Extension; + +#[derive(Debug)] +pub struct ExprData { + is_canonical: AtomicBool, + pos: Pos, + kind: RwLock, +} #[derive(Clone, Debug)] -pub struct Expr { - is_canonical: Arc, - pos: Pos, - kind: Arc>, -} +pub struct Expr(Rc); impl Expr { - pub fn pos(&self) -> Pos { self.pos.clone() } + pub fn pos(&self) -> Pos { self.0.pos.clone() } pub fn as_atom(&self) -> Option { todo!() } pub fn strong_count(&self) -> usize { todo!() } pub fn id(&self) -> api::ExprTicket { api::ExprTicket( - NonZeroU64::new(self.kind.as_ref() as *const RwLock<_> as usize as u64) + NonZeroU64::new(self.0.as_ref() as *const ExprData as usize as u64) .expect("this is a ref, it cannot be null"), ) } - pub fn canonicalize(&self) -> api::ExprTicket { - if !self.is_canonical.swap(true, Ordering::Relaxed) { - KNOWN_EXPRS.write().unwrap().entry(self.id()).or_insert_with(|| self.clone()); - } - self.id() - } - pub fn resolve(tk: api::ExprTicket) -> Option { - KNOWN_EXPRS.read().unwrap().get(&tk).cloned() - } - pub fn from_api(api: &api::Expression, ctx: &mut ExprParseCtx) -> Self { + // pub fn canonicalize(&self) -> api::ExprTicket { + // if !self.is_canonical.swap(true, Ordering::Relaxed) { + // KNOWN_EXPRS.write().unwrap().entry(self.id()).or_insert_with(|| + // self.clone()); } + // self.id() + // } + // pub fn resolve(tk: api::ExprTicket) -> Option { + // KNOWN_EXPRS.read().unwrap().get(&tk).cloned() + // } + pub async fn from_api(api: &api::Expression, ctx: &mut ExprParseCtx) -> Self { if let api::ExpressionKind::Slot(tk) = &api.kind { - return Self::resolve(*tk).expect("Invalid slot"); - } - Self { - kind: Arc::new(RwLock::new(ExprKind::from_api(&api.kind, ctx))), - is_canonical: Arc::default(), - pos: Pos::from_api(&api.location), + return ctx.exprs().get_expr(*tk).expect("Invalid slot"); } + let pos = Pos::from_api(&api.location, &ctx.ctx().i).await; + let kind = RwLock::new(ExprKind::from_api(&api.kind, pos.clone(), ctx).boxed_local().await); + Self(Rc::new(ExprData { is_canonical: AtomicBool::new(false), pos, kind })) } - pub fn to_api(&self) -> api::InspectedKind { + pub async fn to_api(&self) -> api::InspectedKind { use api::InspectedKind as K; - match &*self.kind.read().unwrap() { - ExprKind::Atom(a) => K::Atom(a.to_api()), + match &*self.0.kind.read().await { + ExprKind::Atom(a) => K::Atom(a.to_api().await), ExprKind::Bottom(b) => K::Bottom(b.to_api()), _ => K::Opaque, } } } -impl Drop for Expr { - fn drop(&mut self) { - // If the only two references left are this and known, remove from known - if Arc::strong_count(&self.kind) == 2 && self.is_canonical.load(Ordering::Relaxed) { - // if known is poisoned, a leak is preferable to a panicking destructor - if let Ok(mut w) = KNOWN_EXPRS.write() { - w.remove(&self.id()); - } - } - } -} - -lazy_static! { - static ref KNOWN_EXPRS: RwLock> = RwLock::default(); -} #[derive(Clone, Debug)] pub enum ExprKind { @@ -87,16 +74,20 @@ pub enum ExprKind { Const(Sym), } impl ExprKind { - pub fn from_api(api: &api::ExpressionKind, ctx: &mut ExprParseCtx) -> Self { + pub async fn from_api(api: &api::ExpressionKind, pos: Pos, ctx: &mut ExprParseCtx) -> Self { match_mapping!(api, api::ExpressionKind => ExprKind { - Lambda(id => PathSet::from_api(*id, api), b => Expr::from_api(b, ctx)), - Bottom(b => OrcErrv::from_api(b)), - Call(f => Expr::from_api(f, ctx), x => Expr::from_api(x, ctx)), - Const(c => Sym::from_api(*c)), - Seq(a => Expr::from_api(a, ctx), b => Expr::from_api(b, ctx)), + Lambda(id => PathSet::from_api(*id, api), b => Expr::from_api(b, ctx).await), + Bottom(b => OrcErrv::from_api(b, &ctx.ctx().i).await), + Call(f => Expr::from_api(f, ctx).await, x => Expr::from_api(x, ctx).await), + Const(c => Sym::from_api(*c, &ctx.ctx().i).await), + Seq(a => Expr::from_api(a, ctx).await, b => Expr::from_api(b, ctx).await), } { api::ExpressionKind::Arg(_) => ExprKind::Arg, - api::ExpressionKind::NewAtom(a) => ExprKind::Atom(AtomHand::from_api(a.clone())), + api::ExpressionKind::NewAtom(a) => ExprKind::Atom(AtomHand::from_api( + a, + pos, + &mut ctx.ctx().clone() + ).await), api::ExpressionKind::Slot(_) => panic!("Handled in Expr"), }) } @@ -139,3 +130,8 @@ impl PathSet { } } } + +pub struct WeakExpr(Weak); +impl WeakExpr { + pub fn upgrade(&self) -> Option { self.0.upgrade().map(Expr) } +} diff --git a/orchid-host/src/expr_store.rs b/orchid-host/src/expr_store.rs new file mode 100644 index 0000000..35042eb --- /dev/null +++ b/orchid-host/src/expr_store.rs @@ -0,0 +1,35 @@ +use std::cell::RefCell; +use std::fmt; + +use hashbrown::HashMap; +use hashbrown::hash_map::Entry; + +use crate::api; +use crate::expr::Expr; + +#[derive(Default)] +pub struct ExprStore(RefCell>); +impl ExprStore { + pub fn give_expr(&self, expr: Expr) { + match self.0.borrow_mut().entry(expr.id()) { + Entry::Occupied(mut oe) => oe.get_mut().0 += 1, + Entry::Vacant(v) => { + v.insert((1, expr)); + }, + } + } + pub fn take_expr(&self, ticket: api::ExprTicket) { + (self.0.borrow_mut().entry(ticket)) + .and_replace_entry_with(|_, (rc, rt)| (1 < rc).then_some((rc - 1, rt))); + } + pub fn get_expr(&self, ticket: api::ExprTicket) -> Option { + self.0.borrow().get(&ticket).map(|(_, expr)| expr.clone()) + } +} +impl fmt::Display for ExprStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let r = self.0.borrow(); + let rc: u32 = r.values().map(|v| v.0).sum(); + write!(f, "Store holding {rc} refs to {} exprs", r.len()) + } +} diff --git a/orchid-host/src/extension.rs b/orchid-host/src/extension.rs index 5ac5476..15f09bd 100644 --- a/orchid-host/src/extension.rs +++ b/orchid-host/src/extension.rs @@ -1,111 +1,35 @@ -use std::collections::VecDeque; -use std::num::NonZero; -use std::ops::Deref; -use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering}; -use std::sync::mpsc::{SyncSender, sync_channel}; -use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; -use std::{fmt, io, thread}; +use std::cell::RefCell; +use std::future::Future; +use std::io; +use std::num::NonZeroU64; +use std::rc::{Rc, Weak}; +use async_std::channel::{self, Sender}; +use async_std::sync::Mutex; use derive_destructure::destructure; +use futures::FutureExt; +use futures::future::{join, join_all}; +use futures::task::LocalSpawnExt; use hashbrown::HashMap; -use hashbrown::hash_map::Entry; use itertools::Itertools; -use lazy_static::lazy_static; +use orchid_api::HostMsgSet; use orchid_api_traits::Request; -use orchid_base::builtin::{ExtFactory, ExtPort}; -use orchid_base::char_filter::char_filter_match; +use orchid_base::builtin::ExtInit; use orchid_base::clone; -use orchid_base::error::{OrcErrv, OrcRes}; -use orchid_base::interner::{Tok, intern}; -use orchid_base::location::Pos; +use orchid_base::interner::Tok; use orchid_base::logging::Logger; use orchid_base::macros::mtreev_from_api; -use orchid_base::parse::Comment; use orchid_base::reqnot::{ReqNot, Requester as _}; -use orchid_base::tree::{AtomRepr, ttv_from_api}; -use ordered_float::NotNan; -use substack::{Stackframe, Substack}; +use orchid_base::tree::AtomRepr; use crate::api; -use crate::expr::Expr; +use crate::atom::AtomHand; +use crate::ctx::Ctx; +use crate::expr_store::ExprStore; use crate::macros::{macro_recur, macro_treev_to_api}; -use crate::tree::{Member, ParsTokTree}; +use crate::system::SystemCtor; -#[derive(Debug, destructure)] -pub struct AtomData { - owner: System, - drop: Option, - data: Vec, -} -impl AtomData { - fn api(self) -> api::Atom { - let (owner, drop, data) = self.destructure(); - api::Atom { data, drop, owner: owner.id() } - } - fn api_ref(&self) -> api::Atom { - api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() } - } -} -impl Drop for AtomData { - fn drop(&mut self) { - if let Some(id) = self.drop { - self.owner.reqnot().notify(api::AtomDrop(self.owner.id(), id)) - } - } -} - -#[derive(Clone, Debug)] -pub struct AtomHand(Arc); -impl AtomHand { - pub fn from_api(atom: api::Atom) -> Self { - fn create_new(api::Atom { data, drop, owner }: api::Atom) -> AtomHand { - let owner = System::resolve(owner).expect("Atom owned by non-existing system"); - AtomHand(Arc::new(AtomData { data, drop, owner })) - } - if let Some(id) = atom.drop { - lazy_static! { - static ref OWNED_ATOMS: Mutex>> = - Mutex::default(); - } - let owner = atom.owner; - let mut owned_g = OWNED_ATOMS.lock().unwrap(); - if let Some(data) = owned_g.get(&(owner, id)) { - if let Some(atom) = data.upgrade() { - return Self(atom); - } - } - let new = create_new(atom); - owned_g.insert((owner, id), Arc::downgrade(&new.0)); - new - } else { - create_new(atom) - } - } - pub fn call(self, arg: Expr) -> api::Expression { - let owner_sys = self.0.owner.clone(); - let reqnot = owner_sys.reqnot(); - let ticket = owner_sys.give_expr(arg.canonicalize(), || arg); - match Arc::try_unwrap(self.0) { - Ok(data) => reqnot.request(api::FinalCall(data.api(), ticket)), - Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), ticket)), - } - } - pub fn req(&self, key: api::TStrv, req: Vec) -> Option> { - self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)) - } - pub fn api_ref(&self) -> api::Atom { self.0.api_ref() } - pub fn print(&self) -> String { self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())) } -} -impl AtomRepr for AtomHand { - type Ctx = (); - fn from_api(atom: &orchid_api::Atom, _: Pos, (): &mut Self::Ctx) -> Self { - Self::from_api(atom.clone()) - } - fn to_api(&self) -> orchid_api::Atom { self.api_ref() } -} -impl fmt::Display for AtomHand { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.print()) } -} +pub struct ReqPair(R, Sender); /// Data held about an Extension. This is refcounted within [Extension]. It's /// important to only ever access parts of this struct through the [Arc] because @@ -113,320 +37,194 @@ impl fmt::Display for AtomHand { /// upgrading fails. #[derive(destructure)] pub struct ExtensionData { - port: Mutex>, - // child: Mutex, - // child_stdin: Mutex, + ctx: Ctx, + init: ExtInit, reqnot: ReqNot, systems: Vec, logger: Logger, + next_pars: RefCell, + exprs: ExprStore, + lex_recur: Mutex>>>, } impl Drop for ExtensionData { fn drop(&mut self) { self.reqnot.notify(api::HostExtNotif::Exit); } } -fn acq_expr(sys: api::SysId, extk: api::ExprTicket) { - (System::resolve(sys).expect("Expr acq'd by invalid system")) - .give_expr(extk, || Expr::resolve(extk).expect("Invalid expr acq'd")); -} - -fn rel_expr(sys: api::SysId, extk: api::ExprTicket) { - let sys = System::resolve(sys).unwrap(); - let mut exprs = sys.0.exprs.write().unwrap(); - exprs.entry(extk).and_replace_entry_with(|_, (rc, rt)| { - (0 < rc.fetch_sub(1, Ordering::Relaxed)).then_some((rc, rt)) - }); -} - #[derive(Clone)] -pub struct Extension(Arc); +pub struct Extension(Rc); impl Extension { - pub fn new(fac: Box, logger: Logger) -> io::Result { - Ok(Self(Arc::new_cyclic(|weak: &Weak| { - let (eh, port) = fac.run(Box::new(clone!(weak; move |msg| { - weak.upgrade().inspect(|xd| xd.reqnot.receive(msg)); - }))); - ExtensionData { - systems: (eh.systems.iter().cloned()) - .map(|decl| SystemCtor { decl, ext: weak.clone() }) - .collect(), - logger, - port: Mutex::new(port), - reqnot: ReqNot::new( - clone!(weak; move |sfn, _| { - let data = weak.upgrade().unwrap(); - data.logger.log_buf("Downsending", sfn); - data.port.lock().unwrap().send(sfn); - }), - clone!(weak; move |notif, _| match notif { - api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => acq_expr(acq.0, acq.1), - api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => rel_expr(rel.0, rel.1), + pub fn new(init: ExtInit, logger: Logger, ctx: Ctx) -> io::Result { + Ok(Self(Rc::new_cyclic(|weak: &Weak| ExtensionData { + exprs: ExprStore::default(), + ctx: ctx.clone(), + systems: (init.systems.iter().cloned()) + .map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) }) + .collect(), + logger, + init, + next_pars: RefCell::new(NonZeroU64::new(1).unwrap()), + lex_recur: Mutex::default(), + reqnot: ReqNot::new( + clone!(weak; move |sfn, _| clone!(weak; async move { + let data = weak.upgrade().unwrap(); + data.logger.log_buf("Downsending", sfn); + data.init.send(sfn).await + }.boxed_local())), + clone!(weak; move |notif, _| clone!(weak; async move { + let this = Extension(weak.upgrade().unwrap()); + match notif { + api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => { + let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket"); + this.0.exprs.give_expr(target) + } + api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => { + this.assert_own_sys(rel.0).await; + this.0.exprs.take_expr(rel.1) + } api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => { - acq_expr(mov.inc, mov.expr); - rel_expr(mov.dec, mov.expr); + this.assert_own_sys(mov.dec).await; + let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id"); + let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket"); + recp.ext().0.exprs.give_expr(expr); + this.0.exprs.take_expr(mov.expr); }, - api::ExtHostNotif::Log(api::Log(str)) => weak.upgrade().unwrap().logger.log(str), - }), - |hand, req| match req { - api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()), - api::ExtHostReq::IntReq(intreq) => match intreq { - api::IntReq::InternStr(s) => hand.handle(&s, &intern(&**s.0).to_api()), - api::IntReq::InternStrv(v) => hand.handle(&v, &intern(&*v.0).to_api()), - api::IntReq::ExternStr(si) => hand.handle(&si, &Tok::::from_api(si.0).arc()), - api::IntReq::ExternStrv(vi) => hand.handle( - &vi, - &Arc::new( - Tok::>>::from_api(vi.0).iter().map(|t| t.to_api()).collect_vec(), - ), - ), - }, - api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => { - let sys = System::resolve(atom.owner).unwrap(); - hand.handle(fw, &sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone()))) - }, - api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => { - let sys = System::resolve(id).unwrap(); - hand.handle(fw, &sys.request(body.clone())) - }, - api::ExtHostReq::SubLex(sl) => { - let (rep_in, rep_out) = sync_channel(0); - let lex_g = LEX_RECUR.lock().unwrap(); - let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid"); - req_in.send(ReqPair(sl.clone(), rep_in)).unwrap(); - hand.handle(&sl, &rep_out.recv().unwrap()) - }, - api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => { - let expr = Expr::resolve(target).expect("Invalid ticket"); - hand.handle(&ins, &api::Inspected { - refcount: expr.strong_count() as u32, - location: expr.pos().to_api(), - kind: expr.to_api(), - }) - }, - api::ExtHostReq::RunMacros(ref rm @ api::RunMacros { ref run_id, ref query }) => hand - .handle( - rm, - ¯o_recur( - *run_id, - mtreev_from_api(query, &mut |_| panic!("Recursion never contains atoms")), - ) - .map(|x| macro_treev_to_api(*run_id, x)), - ), - }, - ), - } + api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str), + } + }.boxed_local())), + { + clone!(weak, ctx); + move |hand, req| { + clone!(weak, ctx); + async move { + let this = Self(weak.upgrade().unwrap()); + let i = this.ctx().i.clone(); + match req { + api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await, + api::ExtHostReq::IntReq(intreq) => match intreq { + api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await, + api::IntReq::InternStrv(v) => { + let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await; + hand.handle(&v, &i.i(&tokens).await.to_api()).await + }, + api::IntReq::ExternStr(si) => + hand.handle(&si, &Tok::::from_api(si.0, &i).await.rc()).await, + api::IntReq::ExternStrv(vi) => { + let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec(); + hand.handle(&vi, &markerv).await + }, + }, + api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => { + let sys = ctx.system_inst(atom.owner).await.expect("owner of live atom dropped"); + let reply = + sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await; + hand.handle(fw, &reply).await + }, + api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => { + let sys = ctx.system_inst(id).await.unwrap(); + hand.handle(fw, &sys.request(body.clone()).await).await + }, + api::ExtHostReq::SubLex(sl) => { + let (rep_in, rep_out) = channel::bounded(0); + let lex_g = this.0.lex_recur.lock().await; + let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid"); + req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap(); + hand.handle(&sl, &rep_out.recv().await.unwrap()).await + }, + api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => { + let expr = this.exprs().get_expr(target).expect("Invalid ticket"); + hand + .handle(&ins, &api::Inspected { + refcount: expr.strong_count() as u32, + location: expr.pos().to_api(), + kind: expr.to_api().await, + }) + .await + }, + api::ExtHostReq::RunMacros(ref rm @ api::RunMacros { ref run_id, ref query }) => { + let mtreev = + mtreev_from_api(query, &i, &mut |_| panic!("Atom in macro recur")).await; + match macro_recur(*run_id, mtreev).await { + Some(x) => hand.handle(rm, &Some(macro_treev_to_api(*run_id, x).await)).await, + None => hand.handle(rm, &None).await, + } + }, + api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => + hand.handle(eap, &AtomHand::new(atom.clone(), &ctx).await.print().await).await, + } + } + .boxed_local() + } + }, + ), }))) } - pub fn systems(&self) -> impl Iterator { self.0.systems.iter() } -} - -pub struct SystemCtor { - decl: api::SystemDecl, - ext: Weak, -} -impl SystemCtor { - pub fn name(&self) -> &str { &self.decl.name } - pub fn priority(&self) -> NotNan { self.decl.priority } - pub fn depends(&self) -> impl ExactSizeIterator { - self.decl.depends.iter().map(|s| &**s) + pub(crate) fn reqnot(&self) -> &ReqNot { &self.0.reqnot } + pub fn ctx(&self) -> &Ctx { &self.0.ctx } + pub fn logger(&self) -> &Logger { &self.0.logger } + pub fn system_ctors(&self) -> impl Iterator { self.0.systems.iter() } + pub fn exprs(&self) -> &ExprStore { &self.0.exprs } + pub async fn is_own_sys(&self, id: api::SysId) -> bool { + let sys = self.ctx().system_inst(id).await.expect("invalid sender sys id"); + Rc::ptr_eq(&self.0, &sys.ext().0) } - pub fn run<'a>(&self, depends: impl IntoIterator) -> System { - let mut inst_g = SYSTEM_INSTS.write().unwrap(); - let depends = depends.into_iter().map(|si| si.id()).collect_vec(); - debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided"); - let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension"); - static NEXT_ID: AtomicU16 = AtomicU16::new(1); - let id = - api::SysId(NonZero::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped")); - let sys_inst = ext.reqnot.request(api::NewSystem { depends, id, system: self.decl.id }); - let data = System(Arc::new(SystemInstData { - decl_id: self.decl.id, - ext: Extension(ext), - exprs: RwLock::default(), - lex_filter: sys_inst.lex_filter, - const_root: OnceLock::new(), - line_types: sys_inst.line_types.into_iter().map(Tok::from_api).collect(), - id, - })); - let root = (sys_inst.const_root.into_iter()) - .map(|(k, v)| { - Member::from_api( - api::Member { name: k, kind: v }, - Substack::Bottom.push(Tok::from_api(k)), - &data, - ) - }) - .collect_vec(); - data.0.const_root.set(root).unwrap(); - inst_g.insert(id, data.clone()); - data + pub async fn assert_own_sys(&self, id: api::SysId) { + assert!(self.is_own_sys(id).await, "Incoming message impersonates separate system"); } -} - -lazy_static! { - static ref SYSTEM_INSTS: RwLock> = RwLock::default(); - static ref LEX_RECUR: Mutex>>> = - Mutex::default(); -} - -pub struct ReqPair(R, pub SyncSender); - -#[derive(destructure)] -pub struct SystemInstData { - exprs: RwLock>, - ext: Extension, - decl_id: api::SysDeclId, - lex_filter: api::CharFilter, - id: api::SysId, - const_root: OnceLock>, - line_types: Vec>, -} -impl Drop for SystemInstData { - fn drop(&mut self) { - self.ext.0.reqnot.notify(api::SystemDrop(self.id)); - if let Ok(mut g) = SYSTEM_INSTS.write() { - g.remove(&self.id); - } + pub fn next_pars(&self) -> NonZeroU64 { + let mut next_pars = self.0.next_pars.borrow_mut(); + *next_pars = next_pars.checked_add(1).unwrap_or(NonZeroU64::new(1).unwrap()); + *next_pars } -} -#[derive(Clone)] -pub struct System(Arc); -impl System { - pub fn id(&self) -> api::SysId { self.id } - fn resolve(id: api::SysId) -> Option { SYSTEM_INSTS.read().unwrap().get(&id).cloned() } - fn reqnot(&self) -> &ReqNot { &self.0.ext.0.reqnot } - fn give_expr(&self, ticket: api::ExprTicket, get_expr: impl FnOnce() -> Expr) -> api::ExprTicket { - match self.0.exprs.write().unwrap().entry(ticket) { - Entry::Occupied(mut oe) => { - oe.get_mut().0.fetch_add(1, Ordering::Relaxed); - }, - Entry::Vacant(v) => { - v.insert((AtomicU32::new(1), get_expr())); - }, - } - ticket - } - pub fn get_tree(&self, id: api::TreeId) -> api::MemberKind { - self.reqnot().request(api::GetMember(self.0.id, id)) - } - pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() } - pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) } - /// Have this system lex a part of the source. It is assumed that - /// [Self::can_lex] was called and returned true. - pub fn lex( + pub(crate) async fn lex_req>>( &self, source: Tok, pos: u32, - mut r: impl FnMut(u32) -> Option + Send, + sys: api::SysId, + mut r: impl FnMut(u32) -> F, ) -> api::OrcResult> { // get unique lex ID - static LEX_ID: AtomicU64 = AtomicU64::new(1); - let id = api::ParsId(NonZero::new(LEX_ID.fetch_add(1, Ordering::Relaxed)).unwrap()); - thread::scope(|s| { - // create and register channel - let (req_in, req_out) = sync_channel(0); - LEX_RECUR.lock().unwrap().insert(id, req_in); // LEX_RECUR released - // spawn recursion handler which will exit when the sender is collected - s.spawn(move || { - while let Ok(ReqPair(sublex, rep_in)) = req_out.recv() { - rep_in.send(r(sublex.pos)).unwrap() - } - }); - // Pass control to extension - let ret = - self.reqnot().request(api::LexExpr { id, pos, sys: self.id(), text: source.to_api() }); - // collect sender to unblock recursion handler thread before returning - LEX_RECUR.lock().unwrap().remove(&id); - ret.transpose() - }) // exit recursion handler thread - } - pub fn can_parse(&self, line_type: Tok) -> bool { self.line_types.contains(&line_type) } - pub fn line_types(&self) -> impl Iterator> + '_ { - self.line_types.iter().cloned() - } - pub fn parse( - &self, - line: Vec, - exported: bool, - comments: Vec, - ) -> OrcRes> { - let line = line.iter().map(|t| t.to_api(&mut |n, _| match *n {})).collect_vec(); - let comments = comments.iter().map(Comment::to_api).collect_vec(); - let parsed = - (self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line })) - .map_err(|e| OrcErrv::from_api(&e))?; - Ok(ttv_from_api(parsed, &mut ())) - } - pub fn request(&self, req: Vec) -> Vec { - self.reqnot().request(api::SysFwded(self.id(), req)) - } -} -impl fmt::Debug for System { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let ctor = (self.0.ext.0.systems.iter().find(|c| c.decl.id == self.0.decl_id)) - .expect("System instance with no associated constructor"); - write!(f, "System({} @ {} #{})", ctor.decl.name, ctor.decl.priority, self.0.id.0)?; - match self.0.exprs.read() { - Err(_) => write!(f, "expressions unavailable"), - Ok(r) => { - let rc: u32 = r.values().map(|v| v.0.load(Ordering::Relaxed)).sum(); - write!(f, "{rc} refs to {} exprs", r.len()) + let id = api::ParsId(self.next_pars()); + // create and register channel + let (req_in, req_out) = channel::bounded(0); + self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released + let (ret, ()) = join( + async { + let res = + (self.reqnot()).request(api::LexExpr { id, pos, sys, text: source.to_api() }).await; + // collect sender to unblock recursion handler branch before returning + self.0.lex_recur.lock().await.remove(&id); + res }, - } - } -} -impl Deref for System { - type Target = SystemInstData; - fn deref(&self) -> &Self::Target { self.0.as_ref() } -} - -#[derive(Debug, Clone)] -pub enum SysResolvErr { - Loop(Vec), - Missing(String), -} - -pub fn init_systems(tgts: &[String], exts: &[Extension]) -> Result, SysResolvErr> { - let mut to_load = HashMap::<&str, &SystemCtor>::new(); - let mut to_find = tgts.iter().map(|s| s.as_str()).collect::>(); - while let Some(target) = to_find.pop_front() { - if to_load.contains_key(target) { - continue; - } - let ctor = (exts.iter()) - .flat_map(|e| e.systems().filter(|c| c.decl.name == target)) - .max_by_key(|c| c.decl.priority) - .ok_or_else(|| SysResolvErr::Missing(target.to_string()))?; - to_load.insert(target, ctor); - to_find.extend(ctor.decl.depends.iter().map(|s| s.as_str())); - } - let mut to_load_ordered = Vec::new(); - fn walk_deps<'a>( - graph: &mut HashMap<&str, &'a SystemCtor>, - list: &mut Vec<&'a SystemCtor>, - chain: Stackframe<&str>, - ) -> Result<(), SysResolvErr> { - if let Some(ctor) = graph.remove(chain.item) { - // if the above is none, the system is already queued. Missing systems are - // detected above - for dep in ctor.decl.depends.iter() { - if Substack::Frame(chain).iter().any(|c| c == dep) { - let mut circle = vec![dep.to_string()]; - circle.extend(Substack::Frame(chain).iter().map(|s| s.to_string())); - return Err(SysResolvErr::Loop(circle)); + async { + while let Ok(ReqPair(sublex, rep_in)) = req_out.recv().await { + (rep_in.send(r(sublex.pos).await).await) + .expect("Response channel dropped while request pending") } - walk_deps(graph, list, Substack::Frame(chain).new_frame(dep))? - } - list.push(ctor); - } - Ok(()) + }, + ) + .await; + ret.transpose() } - for tgt in tgts { - walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?; + pub async fn recv_one(&self) { + let reqnot = self.0.reqnot.clone(); + self + .0 + .init + .recv(Box::new(move |msg| async move { reqnot.receive(msg).await }.boxed_local())) + .await; } - let mut systems = HashMap::<&str, System>::new(); - for ctor in to_load_ordered.iter() { - let sys = ctor.run(ctor.depends().map(|n| &systems[n])); - systems.insert(ctor.name(), sys); + pub fn system_drop(&self, id: api::SysId) { + let rc = self.clone(); + (self.ctx().spawn.spawn_local(async move { + rc.reqnot().notify(api::SystemDrop(id)).await; + rc.ctx().systems.write().await.remove(&id); + })) + .expect("Failed to drop system!"); } - Ok(systems.into_values().collect_vec()) + pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) } +} + +pub struct WeakExtension(Weak); +impl WeakExtension { + pub fn upgrade(&self) -> Option { self.0.upgrade().map(Extension) } } diff --git a/orchid-host/src/lex.rs b/orchid-host/src/lex.rs index a715e46..5a5412d 100644 --- a/orchid-host/src/lex.rs +++ b/orchid-host/src/lex.rs @@ -1,18 +1,22 @@ use std::num::NonZeroU64; use std::sync::Arc; +use async_std::sync::Mutex; +use futures::FutureExt; use hashbrown::HashMap; use orchid_base::error::{OrcErrv, OrcRes, mk_errv}; -use orchid_base::interner::{Tok, intern}; +use orchid_base::interner::Tok; use orchid_base::location::Pos; +use orchid_base::match_mapping; use orchid_base::number::{num_to_err, parse_num}; use orchid_base::parse::{name_char, name_start, op_char, unrep_space}; use orchid_base::tokens::PARENS; -use orchid_base::tree::Ph; -use orchid_base::{intern, match_mapping}; +use orchid_base::tree::{AtomRepr, Ph}; use crate::api; -use crate::extension::{AtomHand, System}; +use crate::atom::AtomHand; +use crate::ctx::Ctx; +use crate::system::System; use crate::tree::{ParsTok, ParsTokTree}; pub struct LexCtx<'a> { @@ -20,6 +24,7 @@ pub struct LexCtx<'a> { pub source: &'a Tok, pub tail: &'a str, pub sub_trees: &'a mut HashMap, + pub ctx: &'a Ctx, } impl<'a> LexCtx<'a> { pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b> @@ -29,6 +34,7 @@ impl<'a> LexCtx<'a> { tail: &self.source[pos as usize..], systems: self.systems, sub_trees: &mut *self.sub_trees, + ctx: &self.ctx, } } pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 } @@ -70,7 +76,7 @@ impl<'a> LexCtx<'a> { } } -pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { +pub async fn lex_once<'a>(ctx: &mut LexCtx<'a>) -> OrcRes { let start = ctx.get_pos(); assert!( !ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space), @@ -82,11 +88,13 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { } else if ctx.strip_prefix("::") { ParsTok::NS } else if ctx.strip_prefix("--[") { - let (cmt, tail) = ctx.tail.split_once("]--").ok_or_else(|| { - mk_errv(intern!(str: "Unterminated block comment"), "This block comment has no ending ]--", [ - Pos::Range(start..start + 3).into(), - ]) - })?; + let Some((cmt, tail)) = ctx.tail.split_once("]--") else { + return Err(mk_errv( + ctx.ctx.i.i("Unterminated block comment").await, + "This block comment has no ending ]--", + [Pos::Range(start..start + 3).into()], + )); + }; ctx.set_tail(tail); ParsTok::Comment(Arc::new(cmt.to_string())) } else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) { @@ -99,12 +107,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { while !ctx.strip_char('.') { if ctx.tail.is_empty() { return Err(mk_errv( - intern!(str: "Unclosed lambda"), + ctx.ctx.i.i("Unclosed lambda").await, "Lambdae started with \\ should separate arguments from body with .", [Pos::Range(start..start + 1).into()], )); } - arg.push(lex_once(ctx)?); + arg.push(lex_once(ctx).boxed_local().await?); ctx.trim_ws(); } ParsTok::LambdaHead(arg) @@ -114,12 +122,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { while !ctx.strip_char(*rp) { if ctx.tail.is_empty() { return Err(mk_errv( - intern!(str: "unclosed paren"), + ctx.ctx.i.i("unclosed paren").await, format!("this {lp} has no matching {rp}"), [Pos::Range(start..start + 1).into()], )); } - body.push(lex_once(ctx)?); + body.push(lex_once(ctx).boxed_local().await?); ctx.trim_ws(); } ParsTok::S(*paren, body) @@ -130,8 +138,10 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { if ctx.strip_char('(') { let pos = ctx.get_pos(); let numstr = ctx.get_start_matches(|x| x != ')').trim(); - let num = parse_num(numstr).map_err(|e| num_to_err(e, pos))?; - ParsTok::Macro(Some(num.to_f64())) + match parse_num(numstr) { + Ok(num) => ParsTok::Macro(Some(num.to_f64())), + Err(e) => return Err(num_to_err(e, pos, &*ctx.ctx.i).await.into()), + } } else { ParsTok::Macro(None) } @@ -139,17 +149,26 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { for sys in ctx.systems { let mut errors = Vec::new(); if ctx.tail.starts_with(|c| sys.can_lex(c)) { - let lx = - sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| match lex_once(&mut ctx.push(pos)) { - Ok(t) => Some(api::SubLexed { pos, ticket: ctx.add_subtree(t) }), - Err(e) => { - errors.push(e); - None - }, - }); + let (source, pos) = (ctx.source.clone(), ctx.get_pos()); + let ctx_lck = &Mutex::new(&mut *ctx); + let errors_lck = &Mutex::new(&mut errors); + let lx = sys + .lex(source, pos, |pos| async move { + match lex_once(&mut ctx_lck.lock().await.push(pos)).boxed_local().await { + Ok(t) => Some(api::SubLexed { pos, ticket: ctx_lck.lock().await.add_subtree(t) }), + Err(e) => { + errors_lck.lock().await.push(e); + None + }, + } + }) + .await; match lx { - Err(e) => return Err(errors.into_iter().fold(OrcErrv::from_api(&e), |a, b| a + b)), - Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos))), + Err(e) => + return Err( + errors.into_iter().fold(OrcErrv::from_api(&e, &*ctx.ctx.i).await, |a, b| a + b), + ), + Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos)).await), Ok(None) => match errors.into_iter().reduce(|a, b| a + b) { Some(errors) => return Err(errors), None => continue, @@ -158,12 +177,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { } } if ctx.tail.starts_with(name_start) { - ParsTok::Name(intern(ctx.get_start_matches(name_char))) + ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(name_char)).await) } else if ctx.tail.starts_with(op_char) { - ParsTok::Name(intern(ctx.get_start_matches(op_char))) + ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(op_char)).await) } else { return Err(mk_errv( - intern!(str: "Unrecognized character"), + ctx.ctx.i.i("Unrecognized character").await, "The following syntax is meaningless.", [Pos::Range(start..start + 1).into()], )); @@ -172,16 +191,18 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes { Ok(ParsTokTree { tok, range: start..ctx.get_pos() }) } -fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree { +async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree { let tok = match_mapping!(&api.token, api::Token => ParsTok { - Atom(atom => AtomHand::from_api(atom.clone())), - Bottom(err => OrcErrv::from_api(err)), - LambdaHead(arg => ttv_to_owned(arg, ctx)), - Name(name => Tok::from_api(*name)), - S(p.clone(), b.iter().map(|t| tt_to_owned(t, ctx)).collect()), + Atom(atom => + AtomHand::from_api(atom, Pos::Range(api.range.clone()), &mut ctx.ctx.clone()).await + ), + Bottom(err => OrcErrv::from_api(err, &*ctx.ctx.i).await), + LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await), + Name(name => Tok::from_api(*name, &*ctx.ctx.i).await), + S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await), BR, NS, Comment(c.clone()), - Ph(ph => Ph::from_api(ph)), + Ph(ph => Ph::from_api(ph, &*ctx.ctx.i).await), Macro(*prio), } { api::Token::Slot(id) => return ctx.rm_subtree(*id), @@ -189,20 +210,24 @@ fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree { ParsTokTree { range: api.range.clone(), tok } } -fn ttv_to_owned<'a>( +async fn ttv_to_owned<'a>( api: impl IntoIterator, ctx: &mut LexCtx<'_>, ) -> Vec { - api.into_iter().map(|t| tt_to_owned(t, ctx)).collect() + let mut out = Vec::new(); + for tt in api { + out.push(tt_to_owned(&tt, ctx).await) + } + out } -pub fn lex(text: Tok, systems: &[System]) -> OrcRes> { +pub async fn lex(text: Tok, systems: &[System], ctx: &Ctx) -> OrcRes> { let mut sub_trees = HashMap::new(); - let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems }; + let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, ctx }; let mut tokv = Vec::new(); ctx.trim(unrep_space); while !ctx.tail.is_empty() { - tokv.push(lex_once(&mut ctx)?); + tokv.push(lex_once(&mut ctx).await?); ctx.trim(unrep_space); } Ok(tokv) diff --git a/orchid-host/src/lib.rs b/orchid-host/src/lib.rs index 5d16e2b..26a55c2 100644 --- a/orchid-host/src/lib.rs +++ b/orchid-host/src/lib.rs @@ -1,11 +1,14 @@ use orchid_api as api; -pub mod child; +pub mod atom; +pub mod ctx; pub mod expr; +pub mod expr_store; pub mod extension; pub mod lex; pub mod macros; pub mod parse; pub mod rule; pub mod subprocess; +pub mod system; pub mod tree; diff --git a/orchid-host/src/macros.rs b/orchid-host/src/macros.rs index 62e37b1..3d27d4a 100644 --- a/orchid-host/src/macros.rs +++ b/orchid-host/src/macros.rs @@ -1,14 +1,16 @@ -use std::sync::{Arc, RwLock}; +use std::rc::Rc; +use async_std::sync::RwLock; +use futures::FutureExt; use hashbrown::{HashMap, HashSet}; use itertools::Itertools; -use lazy_static::lazy_static; +use orchid_base::interner::Interner; use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api}; use orchid_base::name::Sym; use trait_set::trait_set; use crate::api; -use crate::extension::AtomHand; +use crate::atom::AtomHand; use crate::rule::matcher::{NamedMatcher, PriodMatcher}; use crate::rule::state::MatchState; use crate::tree::Code; @@ -17,38 +19,41 @@ pub type MacTok = MTok<'static, AtomHand>; pub type MacTree = MTree<'static, AtomHand>; trait_set! { - trait MacroCB = Fn(Vec) -> Option> + Send + Sync; + trait MacroCB = Fn(Vec) -> Option>; } -lazy_static! { - static ref RECURSION: RwLock>> = RwLock::default(); - static ref MACRO_SLOTS: RwLock>>> = +thread_local! { + static RECURSION: RwLock>> = RwLock::default(); + static MACRO_SLOTS: RwLock>>> = RwLock::default(); } -pub fn macro_recur(run_id: api::ParsId, input: Vec) -> Option> { +pub async fn macro_recur(run_id: api::ParsId, input: Vec) -> Option> { (RECURSION.read().unwrap()[&run_id])(input) } -pub fn macro_treev_to_api(run_id: api::ParsId, mtree: Vec) -> Vec { +pub async fn macro_treev_to_api(run_id: api::ParsId, mtree: Vec) -> Vec { let mut g = MACRO_SLOTS.write().unwrap(); let run_cache = g.get_mut(&run_id).expect("Parser run not found"); mtreev_to_api(&mtree, &mut |a: &AtomHand| { let id = api::MacroTreeId((run_cache.len() as u64 + 1).try_into().unwrap()); - run_cache.insert(id, Arc::new(MacTok::Atom(a.clone()))); + run_cache.insert(id, Rc::new(MacTok::Atom(a.clone()))); api::MacroToken::Slot(id) }) } -pub fn macro_treev_from_api(api: Vec) -> Vec { - mtreev_from_api(&api, &mut |atom| MacTok::Atom(AtomHand::from_api(atom.clone()))) +pub async fn macro_treev_from_api(api: Vec, i: &Interner) -> Vec { + mtreev_from_api(&api, i, &mut |atom| { + async { MacTok::Atom(AtomHand::from_api(atom.clone())) }.boxed_local() + }) + .await } pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option> { let mut slots = (MACRO_SLOTS.write().unwrap()).remove(&run_id).expect("Run not found"); return work(&mut slots, tree); fn work( - slots: &mut HashMap>, + slots: &mut HashMap>, tree: &[MacTree], ) -> Option> { let items = (tree.iter()) @@ -59,8 +64,8 @@ pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option panic!("Ref is an extension-local optimization"), MacTok::Done(_) => panic!("Created and removed by matcher"), MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(), - MacTok::S(paren, b) => Arc::new(MacTok::S(*paren, work(slots, b)?)), - MacTok::Lambda(a, b) => Arc::new(match (work(slots, a), work(slots, b)) { + MacTok::S(paren, b) => Rc::new(MacTok::S(*paren, work(slots, b)?)), + MacTok::Lambda(a, b) => Rc::new(match (work(slots, a), work(slots, b)) { (None, None) => return None, (Some(a), None) => MacTok::Lambda(a, b.clone()), (None, Some(b)) => MacTok::Lambda(a.clone(), b), @@ -92,7 +97,7 @@ pub struct MacroRepo { impl MacroRepo { /// TODO: the recursion inside this function needs to be moved into Orchid. /// See the markdown note - pub fn process_exprv(&self, target: &[MacTree]) -> Option> { + pub fn process_exprv(&self, target: &[MacTree], i: &Interner) -> Option> { let mut workcp = target.to_vec(); let mut lexicon; @@ -100,24 +105,25 @@ impl MacroRepo { lexicon = HashSet::new(); target.iter().for_each(|tgt| fill_lexicon(tgt, &mut lexicon)); - for (i, tree) in workcp.iter().enumerate() { + for (idx, tree) in workcp.iter().enumerate() { let MacTok::Name(name) = &*tree.tok else { continue }; let matches = (self.named.get(name).into_iter().flatten()) .filter(|m| m.deps.is_subset(&lexicon)) .filter_map(|mac| { - mac.cases.iter().find_map(|cas| cas.0.apply(&workcp[i..], |_| false).map(|s| (cas, s))) + (mac.cases.iter()) + .find_map(|cas| cas.0.apply(&workcp[idx..], i, |_| false).map(|s| (cas, s))) }) .collect_vec(); assert!( matches.len() < 2, "Multiple conflicting matches on {:?}: {:?}", - &workcp[i..], + &workcp[idx..], matches ); let Some((case, (state, tail))) = matches.into_iter().next() else { continue }; let inj = (run_body(&case.1, state).into_iter()) - .map(|MacTree { pos, tok }| MacTree { pos, tok: Arc::new(MacTok::Done(tok)) }); - workcp.splice(i..(workcp.len() - tail.len()), inj); + .map(|MacTree { pos, tok }| MacTree { pos, tok: Rc::new(MacTok::Done(tok)) }); + workcp.splice(idx..(workcp.len() - tail.len()), inj); continue 'try_named; } break; @@ -133,13 +139,14 @@ impl MacroRepo { let results = (workcp.into_iter()) .map(|mt| match &*mt.tok { - MTok::S(p, body) => self.process_exprv(body).map(|body| MTok::S(*p, body).at(mt.pos)), - MTok::Lambda(arg, body) => match (self.process_exprv(arg), self.process_exprv(body)) { - (Some(arg), Some(body)) => Some(MTok::Lambda(arg, body).at(mt.pos)), - (Some(arg), None) => Some(MTok::Lambda(arg, body.to_vec()).at(mt.pos)), - (None, Some(body)) => Some(MTok::Lambda(arg.to_vec(), body).at(mt.pos)), - (None, None) => None, - }, + MTok::S(p, body) => self.process_exprv(body, i).map(|body| MTok::S(*p, body).at(mt.pos)), + MTok::Lambda(arg, body) => + match (self.process_exprv(arg, i), self.process_exprv(body, i)) { + (Some(arg), Some(body)) => Some(MTok::Lambda(arg, body).at(mt.pos)), + (Some(arg), None) => Some(MTok::Lambda(arg, body.to_vec()).at(mt.pos)), + (None, Some(body)) => Some(MTok::Lambda(arg.to_vec(), body).at(mt.pos)), + (None, None) => None, + }, _ => None, }) .collect_vec(); @@ -169,6 +176,6 @@ fn run_body(body: &Code, mut state: MatchState<'_>) -> Vec { let inject: Vec = todo!("Call the interpreter with bindings"); inject .into_iter() - .map(|MTree { pos, tok }| MTree { pos, tok: Arc::new(MTok::Done(tok)) }) + .map(|MTree { pos, tok }| MTree { pos, tok: Rc::new(MTok::Done(tok)) }) .collect_vec() } diff --git a/orchid-host/src/parse.rs b/orchid-host/src/parse.rs index e481ee9..3e6a39b 100644 --- a/orchid-host/src/parse.rs +++ b/orchid-host/src/parse.rs @@ -1,10 +1,10 @@ -use std::sync::Arc; -use std::{iter, thread}; +use std::rc::Rc; +use futures::FutureExt; +use futures::future::join_all; use itertools::Itertools; use never::Never; use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err, mk_errv}; -use orchid_base::intern; use orchid_base::interner::Tok; use orchid_base::location::Pos; use orchid_base::macros::{MTok, MTree}; @@ -16,8 +16,9 @@ use orchid_base::parse::{ use orchid_base::tree::{Paren, TokTree, Token}; use substack::Substack; -use crate::extension::{AtomHand, System}; +use crate::atom::AtomHand; use crate::macros::MacTree; +use crate::system::System; use crate::tree::{ Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, ParsTokTree, Rule, RuleKind, }; @@ -29,111 +30,104 @@ pub trait ParseCtx: Send + Sync { fn reporter(&self) -> &impl Reporter; } -pub fn parse_items( +pub async fn parse_items( ctx: &impl ParseCtx, - path: Substack>, - items: ParsSnippet, + path: Substack<'_, Tok>, + items: ParsSnippet<'_>, ) -> OrcRes> { - let lines = line_items(items); - let mut ok = iter::from_fn(|| None).take(lines.len()).collect_vec(); - thread::scope(|s| { - let mut threads = Vec::new(); - for (slot, Parsed { output: cmts, tail }) in ok.iter_mut().zip(lines.into_iter()) { - let path = &path; - threads.push(s.spawn(move || { - *slot = Some(parse_item(ctx, path.clone(), cmts, tail)?); - Ok::<(), OrcErrv>(()) - })) - } - for t in threads { - t.join().unwrap().err().into_iter().flatten().for_each(|e| ctx.reporter().report(e)) - } - }); - Ok(ok.into_iter().flatten().flatten().collect_vec()) + let lines = line_items(items).await; + let line_res = + join_all(lines.into_iter().map(|p| parse_item(ctx, path.clone(), p.output, p.tail))).await; + Ok(line_res.into_iter().flat_map(|l| l.ok().into_iter().flatten()).collect()) } -pub fn parse_item( +pub async fn parse_item( ctx: &impl ParseCtx, - path: Substack>, + path: Substack<'_, Tok>, comments: Vec, - item: ParsSnippet, + item: ParsSnippet<'_>, ) -> OrcRes> { match item.pop_front() { Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n { - n if *n == intern!(str: "export") => match try_pop_no_fluff(postdisc)? { + n if *n == item.i("export").await => match try_pop_no_fluff(postdisc).await? { Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => - parse_exportable_item(ctx, path, comments, true, n.clone(), tail), + parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await, Parsed { output: TokTree { tok: Token::NS, .. }, tail } => { - let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail)?; + let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail).await?; let mut ok = Vec::new(); - exports.into_iter().for_each(|(e, pos)| match (&e.path.as_slice(), e.name) { - ([], Some(n)) => - ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }), - (_, Some(_)) => ctx.reporter().report(mk_err( - intern!(str: "Compound export"), - "Cannot export compound names (names containing the :: separator)", - [pos.into()], - )), - (_, None) => ctx.reporter().report(mk_err( - intern!(str: "Wildcard export"), - "Exports cannot contain the globstar *", - [pos.into()], - )), - }); - expect_end(tail)?; + for (e, pos) in exports { + match (&e.path.as_slice(), e.name) { + ([], Some(n)) => + ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }), + (_, Some(_)) => ctx.reporter().report(mk_err( + tail.i("Compound export").await, + "Cannot export compound names (names containing the :: separator)", + [pos.into()], + )), + (_, None) => ctx.reporter().report(mk_err( + tail.i("Wildcard export").await, + "Exports cannot contain the globstar *", + [pos.into()], + )), + } + } + expect_end(tail).await?; Ok(ok) }, - Parsed { output, .. } => Err(mk_errv( - intern!(str: "Malformed export"), + Parsed { output, tail } => Err(mk_errv( + tail.i("Malformed export").await, "`export` can either prefix other lines or list names inside ::( ) or ::[ ]", [Pos::Range(output.range.clone()).into()], )), }, - n if *n == intern!(str: "import") => parse_import(ctx, postdisc).map(|v| { + n if *n == item.i("import").await => parse_import(ctx, postdisc).await.map(|v| { Vec::from_iter(v.into_iter().map(|(t, pos)| Item { comments: comments.clone(), pos, kind: ItemKind::Import(t), })) }), - n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc), + n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await, }, Some(_) => - Err(mk_errv(intern!(str: "Expected a line type"), "All lines must begin with a keyword", [ + Err(mk_errv(item.i("Expected a line type").await, "All lines must begin with a keyword", [ Pos::Range(item.pos()).into(), ])), None => unreachable!("These lines are filtered and aggregated in earlier stages"), } } -pub fn parse_import(ctx: &impl ParseCtx, tail: ParsSnippet) -> OrcRes> { - let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail)?; - expect_end(tail)?; +pub async fn parse_import( + ctx: &impl ParseCtx, + tail: ParsSnippet<'_>, +) -> OrcRes> { + let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail).await?; + expect_end(tail).await?; Ok(imports) } -pub fn parse_exportable_item( +pub async fn parse_exportable_item( ctx: &impl ParseCtx, - path: Substack>, + path: Substack<'_, Tok>, comments: Vec, exported: bool, discr: Tok, - tail: ParsSnippet, + tail: ParsSnippet<'_>, ) -> OrcRes> { - let kind = if discr == intern!(str: "mod") { - let (name, body) = parse_module(ctx, path, tail)?; + let kind = if discr == tail.i("mod").await { + let (name, body) = parse_module(ctx, path, tail).await?; ItemKind::Member(Member::new(name, MemberKind::Mod(body))) - } else if discr == intern!(str: "const") { - let (name, val) = parse_const(tail)?; - let locator = CodeLocator::to_const(path.push(name.clone()).unreverse()); + } else if discr == tail.i("const").await { + let (name, val) = parse_const(tail).await?; + let locator = CodeLocator::to_const(tail.i(&path.push(name.clone()).unreverse()).await); ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val)))) } else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) { - let line = sys.parse(tail.to_vec(), exported, comments)?; - return parse_items(ctx, path, Snippet::new(tail.prev(), &line)); + let line = sys.parse(tail.to_vec(), exported, comments).await?; + return parse_items(ctx, path, Snippet::new(tail.prev(), &line, tail.interner())).await; } else { let ext_lines = ctx.systems().flat_map(System::line_types).join(", "); return Err(mk_errv( - intern!(str: "Unrecognized line type"), + tail.i("Unrecognized line type").await, format!("Line types are: const, mod, macro, grammar, {ext_lines}"), [Pos::Range(tail.prev().range.clone()).into()], )); @@ -141,82 +135,90 @@ pub fn parse_exportable_item( Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }]) } -pub fn parse_module( +pub async fn parse_module( ctx: &impl ParseCtx, - path: Substack>, - tail: ParsSnippet, + path: Substack<'_, Tok>, + tail: ParsSnippet<'_>, ) -> OrcRes<(Tok, Module)> { - let (name, tail) = match try_pop_no_fluff(tail)? { + let (name, tail) = match try_pop_no_fluff(tail).await? { Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail), Parsed { output, .. } => { return Err(mk_errv( - intern!(str: "Missing module name"), - format!("A name was expected, {output} was found"), + tail.i("Missing module name").await, + format!("A name was expected, {} was found", output.print().await), [Pos::Range(output.range.clone()).into()], )); }, }; - let Parsed { output, tail: surplus } = try_pop_no_fluff(tail)?; - expect_end(surplus)?; - let body = output.as_s(Paren::Round).ok_or_else(|| { - mk_errv( - intern!(str: "Expected module body"), - format!("A ( block ) was expected, {output} was found"), + let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?; + expect_end(surplus).await?; + let Some(body) = output.as_s(Paren::Round, tail.interner()) else { + return Err(mk_errv( + tail.i("Expected module body").await, + format!("A ( block ) was expected, {} was found", output.print().await), [Pos::Range(output.range.clone()).into()], - ) - })?; + )); + }; let path = path.push(name.clone()); - Ok((name, Module::new(parse_items(ctx, path, body)?))) + Ok((name, Module::new(parse_items(ctx, path, body).await?))) } -pub fn parse_const(tail: ParsSnippet) -> OrcRes<(Tok, Vec)> { - let Parsed { output, tail } = try_pop_no_fluff(tail)?; - let name = output.as_name().ok_or_else(|| { - mk_errv( - intern!(str: "Missing module name"), - format!("A name was expected, {output} was found"), - [Pos::Range(output.range.clone()).into()], - ) - })?; - let Parsed { output, tail } = try_pop_no_fluff(tail)?; - if !output.is_kw(intern!(str: "=")) { +pub async fn parse_const(tail: ParsSnippet<'_>) -> OrcRes<(Tok, Vec)> { + let Parsed { output, tail } = try_pop_no_fluff(tail).await?; + let Some(name) = output.as_name() else { return Err(mk_errv( - intern!(str: "Missing walrus := separator"), - format!("Expected operator := , found {output}"), + tail.i("Missing module name").await, + format!("A name was expected, {} was found", output.print().await), + [Pos::Range(output.range.clone()).into()], + )); + }; + let Parsed { output, tail } = try_pop_no_fluff(tail).await?; + if !output.is_kw(tail.i("=").await) { + return Err(mk_errv( + tail.i("Missing walrus := separator").await, + format!("Expected operator := , found {}", output.print().await), [Pos::Range(output.range.clone()).into()], )); } - try_pop_no_fluff(tail)?; + try_pop_no_fluff(tail).await?; Ok((name, tail.iter().flat_map(strip_fluff).collect_vec())) } -pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes> { +pub async fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes> { let mut mtreev = Vec::new(); while let Some((ttree, tail)) = snip.pop_front() { let (range, tok, tail) = match &ttree.tok { - Token::S(p, b) => - (ttree.range.clone(), MTok::S(*p, parse_mtree(Snippet::new(ttree, b))?), tail), + Token::S(p, b) => ( + ttree.range.clone(), + MTok::S(*p, parse_mtree(Snippet::new(ttree, b, snip.interner())).boxed_local().await?), + tail, + ), Token::Name(tok) => { let mut segments = vec![tok.clone()]; let mut end = ttree.range.end; while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() { - let Parsed { output, tail } = try_pop_no_fluff(tail)?; - segments.push(output.as_name().ok_or_else(|| { - mk_errv( - intern!(str: "Namespaced name interrupted"), + let Parsed { output, tail } = try_pop_no_fluff(tail).await?; + let Some(seg) = output.as_name() else { + return Err(mk_errv( + tail.i("Namespaced name interrupted").await, "In expression context, :: must always be followed by a name.\n\ - ::() is permitted only in import and export items", + ::() is permitted only in import and export items", [Pos::Range(output.range.clone()).into()], - ) - })?); + )); + }; + segments.push(seg); snip = tail; end = output.range.end; } - (ttree.range.start..end, MTok::Name(Sym::new(segments).unwrap()), snip) + ( + ttree.range.start..end, + MTok::Name(Sym::new(segments, snip.interner()).await.unwrap()), + snip, + ) }, Token::NS => { return Err(mk_errv( - intern!(str: "Unexpected :: in macro pattern"), + tail.i("Unexpected :: in macro pattern").await, ":: can only follow a name outside export statements", [Pos::Range(ttree.range.clone()).into()], )); @@ -224,8 +226,11 @@ pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes> { Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail), Token::Atom(_) | Token::Macro(_) => { return Err(mk_errv( - intern!(str: "Unsupported token in macro patterns"), - format!("Macro patterns can only contain names, braces, and lambda, not {ttree}."), + tail.i("Unsupported token in macro patterns").await, + format!( + "Macro patterns can only contain names, braces, and lambda, not {}.", + ttree.print().await + ), [Pos::Range(ttree.range.clone()).into()], )); }, @@ -233,50 +238,57 @@ pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes> { Token::Bottom(e) => return Err(e.clone()), Token::LambdaHead(arg) => ( ttree.range.start..snip.pos().end, - MTok::Lambda(parse_mtree(Snippet::new(ttree, arg))?, parse_mtree(tail)?), - Snippet::new(ttree, &[]), + MTok::Lambda( + parse_mtree(Snippet::new(ttree, arg, snip.interner())).await?, + parse_mtree(tail).await?, + ), + Snippet::new(ttree, &[], snip.interner()), ), - Token::Slot(_) | Token::X(_) => panic!("Did not expect {} in parsed token tree", &ttree.tok), + Token::Slot(_) | Token::X(_) => + panic!("Did not expect {} in parsed token tree", &ttree.tok.print().await), }; - mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Arc::new(tok) }); + mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Rc::new(tok) }); snip = tail; } Ok(mtreev) } -pub fn parse_macro( - tail: ParsSnippet, +pub async fn parse_macro( + tail: ParsSnippet<'_>, macro_i: u16, - path: Substack>, + path: Substack<'_, Tok>, ) -> OrcRes> { - let (surplus, prev, block) = match try_pop_no_fluff(tail)? { + let (surplus, prev, block) = match try_pop_no_fluff(tail).await? { Parsed { tail, output: o @ TokTree { tok: Token::S(Paren::Round, b), .. } } => (tail, o, b), Parsed { output, .. } => { return Err(mk_errv( - intern!(str: "m"), + tail.i("m").await, "Macro blocks must either start with a block or a ..$:number", [Pos::Range(output.range.clone()).into()], )); }, }; - expect_end(surplus)?; + expect_end(surplus).await?; let mut errors = Vec::new(); let mut rules = Vec::new(); - for (i, item) in line_items(Snippet::new(prev, block)).into_iter().enumerate() { - let Parsed { tail, output } = try_pop_no_fluff(item.tail)?; - if !output.is_kw(intern!(str: "rule")) { + for (i, item) in + line_items(Snippet::new(&prev, block, tail.interner())).await.into_iter().enumerate() + { + let Parsed { tail, output } = try_pop_no_fluff(item.tail).await?; + if !output.is_kw(tail.i("rule").await) { errors.extend(mk_errv( - intern!(str: "non-rule in macro"), - format!("Expected `rule`, got {output}"), + tail.i("non-rule in macro").await, + format!("Expected `rule`, got {}", output.print().await), [Pos::Range(output.range.clone()).into()], )); continue; }; - let (pat, body) = match tail.split_once(|t| t.is_kw(intern!(str: "=>"))) { + let arrow = tail.i("=>").await; + let (pat, body) = match tail.split_once(|t| t.is_kw(arrow.clone())) { Some((a, b)) => (a, b), None => { errors.extend(mk_errv( - intern!(str: "no => in macro rule"), + tail.i("no => in macro rule").await, "The pattern and body of a rule must be separated by a =>", [Pos::Range(tail.pos()).into()], )); @@ -286,9 +298,9 @@ pub fn parse_macro( rules.push(Rule { comments: item.output, pos: Pos::Range(tail.pos()), - pattern: parse_mtree(pat)?, + pattern: parse_mtree(pat).await?, kind: RuleKind::Native(Code::from_code( - CodeLocator::to_rule(path.unreverse(), macro_i, i as u16), + CodeLocator::to_rule(tail.i(&path.unreverse()).await, macro_i, i as u16), body.to_vec(), )), }) diff --git a/orchid-host/src/rule/build.rs b/orchid-host/src/rule/build.rs index 8760357..917973d 100644 --- a/orchid-host/src/rule/build.rs +++ b/orchid-host/src/rule/build.rs @@ -107,43 +107,55 @@ fn mk_scalar(pattern: &MacTree) -> ScalMatcher { #[cfg(test)] mod test { - use std::sync::Arc; + use std::rc::Rc; use orchid_api::PhKind; + use orchid_base::interner::Interner; use orchid_base::location::SourceRange; + use orchid_base::sym; use orchid_base::tokens::Paren; use orchid_base::tree::Ph; - use orchid_base::{intern, sym}; + use test_executors::spin_on; use super::mk_any; use crate::macros::{MacTok, MacTree}; #[test] fn test_scan() { - let ex = |tok: MacTok| MacTree { tok: Arc::new(tok), pos: SourceRange::mock().pos() }; - let pattern = vec![ - ex(MacTok::Ph(Ph { - kind: PhKind::Vector { priority: 0, at_least_one: false }, - name: intern!(str: "::prefix"), - })), - ex(MacTok::Name(sym!(prelude::do))), - ex(MacTok::S(Paren::Round, vec![ + spin_on(async { + let i = Interner::new_master(); + let ex = |tok: MacTok| async { + MacTree { tok: Rc::new(tok), pos: SourceRange::mock(&i).await.pos() } + }; + let pattern = vec![ ex(MacTok::Ph(Ph { kind: PhKind::Vector { priority: 0, at_least_one: false }, - name: intern!(str: "expr"), - })), - ex(MacTok::Name(sym!(prelude::;))), + name: i.i("::prefix").await, + })) + .await, + ex(MacTok::Name(sym!(prelude::do; i).await)).await, + ex(MacTok::S(Paren::Round, vec![ + ex(MacTok::Ph(Ph { + kind: PhKind::Vector { priority: 0, at_least_one: false }, + name: i.i("expr").await, + })) + .await, + ex(MacTok::Name(sym!(prelude::; ; i).await)).await, + ex(MacTok::Ph(Ph { + kind: PhKind::Vector { priority: 1, at_least_one: false }, + name: i.i("rest").await, + })) + .await, + ])) + .await, ex(MacTok::Ph(Ph { - kind: PhKind::Vector { priority: 1, at_least_one: false }, - name: intern!(str: "rest"), - })), - ])), - ex(MacTok::Ph(Ph { - kind: PhKind::Vector { priority: 0, at_least_one: false }, - name: intern!(str: "::suffix"), - })), - ]; - let matcher = mk_any(&pattern); - println!("{matcher}"); + kind: PhKind::Vector { priority: 0, at_least_one: false }, + name: i.i("::suffix").await, + })) + .await, + ]; + let matcher = mk_any(&pattern); + println!("{matcher}"); + }) } } diff --git a/orchid-host/src/rule/matcher.rs b/orchid-host/src/rule/matcher.rs index 0d237ca..94e7a88 100644 --- a/orchid-host/src/rule/matcher.rs +++ b/orchid-host/src/rule/matcher.rs @@ -2,7 +2,7 @@ use std::fmt; use itertools::Itertools; use orchid_api::PhKind; -use orchid_base::intern; +use orchid_base::interner::Interner; use orchid_base::location::Pos; use orchid_base::name::Sym; use orchid_base::tree::Ph; @@ -21,7 +21,7 @@ pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwra pub struct NamedMatcher(AnyMatcher); impl NamedMatcher { - pub fn new(pattern: &[MacTree]) -> Self { + pub async fn new(pattern: &[MacTree], i: &Interner) -> Self { assert!( matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))), "Named matchers must begin with a name" @@ -31,7 +31,7 @@ impl NamedMatcher { true => Self(mk_any(pattern)), false => { let kind: PhKind = PhKind::Vector { priority: 0, at_least_one: false }; - let suffix = [MacTok::Ph(Ph { name: intern!(str: "::after"), kind }).at(Pos::None)]; + let suffix = [MacTok::Ph(Ph { name: i.i("::after").await, kind }).at(Pos::None)]; Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec())) }, } @@ -39,18 +39,18 @@ impl NamedMatcher { /// Also returns the tail, if any, which should be matched further /// Note that due to how priod works below, the main usable information from /// the tail is its length - pub fn apply<'a>( + pub async fn apply<'a>( &self, seq: &'a [MacTree], + i: &Interner, save_loc: impl Fn(Sym) -> bool, ) -> Option<(MatchState<'a>, &'a [MacTree])> { - any_match(&self.0, seq, &save_loc).map(|mut state| { - match state.remove(intern!(str: "::after")) { - Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"), - Some(StateEntry::Vec(v)) => (state, v), - None => (state, &[][..]), - } - }) + let mut state = any_match(&self.0, seq, &save_loc)?; + match state.remove(i.i("::after").await) { + Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"), + Some(StateEntry::Vec(v)) => Some((state, v)), + None => Some((state, &[][..])), + } } } impl fmt::Display for NamedMatcher { diff --git a/orchid-host/src/rule/scal_match.rs b/orchid-host/src/rule/scal_match.rs index 2c6a775..98d0bfa 100644 --- a/orchid-host/src/rule/scal_match.rs +++ b/orchid-host/src/rule/scal_match.rs @@ -20,7 +20,7 @@ pub fn scal_match<'a>( (ScalMatcher::Placeh { key }, _) => Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))), (ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 => - any_match(b_mat, &body[..], save_loc), + any_match(&b_mat, &body[..], save_loc), (ScalMatcher::Lambda(arg_mat, b_mat), MacTok::Lambda(arg, body)) => Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)), _ => None, diff --git a/orchid-host/src/subprocess.rs b/orchid-host/src/subprocess.rs index fe03d97..574cb40 100644 --- a/orchid-host/src/subprocess.rs +++ b/orchid-host/src/subprocess.rs @@ -1,79 +1,96 @@ -use std::io::{self, BufRead as _, Write}; +use std::cell::RefCell; use std::path::PathBuf; -use std::sync::mpsc::sync_channel; -use std::{process, thread}; +use std::pin::Pin; +use std::rc::Rc; +use std::thread; +use async_process::{self, Child, ChildStdin, ChildStdout}; +use async_std::io::{self, BufReadExt, BufReader}; use async_std::sync::Mutex; +use futures::FutureExt; +use futures::future::LocalBoxFuture; +use futures::task::LocalSpawnExt; use orchid_api_traits::{Decode, Encode}; use orchid_base::builtin::{ExtInit, ExtPort}; use orchid_base::logging::Logger; use orchid_base::msg::{recv_msg, send_msg}; use crate::api; +use crate::ctx::Ctx; -pub struct ExtensionCommand(pub process::Command, pub Logger); -impl ExtFactory for ExtensionCommand { - fn run(self: Box, onmessage: OnMessage) -> ExtInit { - let Self(mut cmd, logger) = *self; - let prog_pbuf = PathBuf::from(cmd.get_program()); - let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy().to_string(); - let mut child = cmd - .stdin(process::Stdio::piped()) - .stdout(process::Stdio::piped()) - .stderr(process::Stdio::piped()) - .spawn()?; - let mut stdin = child.stdin.take().unwrap(); - api::HostHeader { log_strategy: logger.strat() }.encode(&mut stdin); - stdin.flush()?; - let mut stdout = child.stdout.take().unwrap(); - let header = api::ExtensionHeader::decode(&mut stdout); - let child_stderr = child.stderr.take().unwrap(); - let (set_onmessage, recv_onmessage) = sync_channel(0); - thread::Builder::new().name(format!("stdout-fwd:{prog}")).spawn(move || { - let mut onmessage: Box = recv_onmessage.recv().unwrap(); - drop(recv_onmessage); - loop { - match recv_msg(&mut stdout) { - Ok(msg) => onmessage(&msg[..]), - Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break, - Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()), - } - } - })?; - thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || { - let mut reader = io::BufReader::new(child_stderr); +pub async fn ext_command( + cmd: std::process::Command, + logger: Logger, + ctx: Ctx, +) -> io::Result { + let prog_pbuf = PathBuf::from(cmd.get_program()); + let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy().to_string(); + let mut child = async_process::Command::from(cmd) + .stdin(async_process::Stdio::piped()) + .stdout(async_process::Stdio::piped()) + .stderr(async_process::Stdio::piped()) + .spawn()?; + let mut stdin = child.stdin.take().unwrap(); + api::HostHeader { log_strategy: logger.strat() }.encode(Pin::new(&mut stdin)); + let mut stdout = child.stdout.take().unwrap(); + let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await; + let child_stderr = child.stderr.take().unwrap(); + thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || { + async_std::task::block_on(async move { + let mut reader = BufReader::new(child_stderr); loop { let mut buf = String::new(); - if 0 == reader.read_line(&mut buf).unwrap() { + if 0 == reader.read_line(&mut buf).await.unwrap() { break; } logger.log(buf); } - })?; - Ok(Subprocess { child: Mutex::new(child), stdin: Mutex::new(stdin), set_onmessage, header }) - } + }) + })?; + Ok(ExtInit { + header, + port: Box::new(Subprocess { + child: Rc::new(RefCell::new(child)), + stdin: Mutex::new(Box::pin(stdin)), + stdout: Mutex::new(Box::pin(stdout)), + ctx, + }), + }) } pub struct Subprocess { - child: Mutex, - stdin: Mutex, - stdout: Mutex, - header: api::ExtensionHeader, -} -impl Subprocess { - pub fn new(mut cmd: process::Command, logger: Logger) -> io::Result {} + child: Rc>, + stdin: Mutex>>, + stdout: Mutex>>, + ctx: Ctx, } impl Drop for Subprocess { - fn drop(&mut self) { self.child.lock().unwrap().wait().expect("Extension exited with error"); } + fn drop(&mut self) { + let child = self.child.clone(); + (self.ctx.spawn.spawn_local(async move { + let status = child.borrow_mut().status().await.expect("Extension exited with error"); + assert!(status.success(), "Extension exited with error {status}"); + })) + .expect("Could not spawn process terminating future") + } } impl ExtPort for Subprocess { - fn send(&self, msg: &[u8]) { + fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> { if msg.starts_with(&[0, 0, 0, 0x1c]) { panic!("Received unnecessary prefix"); } - send_msg(&mut *self.stdin.lock().unwrap(), msg).unwrap() + async { send_msg(Pin::new(&mut *self.stdin.lock().await), msg).await.unwrap() }.boxed_local() } - fn recv<'a>(&self, cb: Box) -> futures::future::BoxFuture<()> { - async {} + fn recv<'a>( + &'a self, + cb: Box LocalBoxFuture<'_, ()> + 'a>, + ) -> LocalBoxFuture<'a, ()> { + Box::pin(async { + match recv_msg(self.stdout.lock().await.as_mut()).await { + Ok(msg) => cb(&msg).await, + Err(e) if e.kind() == io::ErrorKind::BrokenPipe => (), + Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()), + } + }) } } diff --git a/orchid-host/src/system.rs b/orchid-host/src/system.rs new file mode 100644 index 0000000..09272a7 --- /dev/null +++ b/orchid-host/src/system.rs @@ -0,0 +1,213 @@ +use std::collections::VecDeque; +use std::fmt; +use std::future::Future; +use std::rc::{Rc, Weak}; + +use async_stream::stream; +use derive_destructure::destructure; +use futures::StreamExt; +use futures::future::join_all; +use futures::task::LocalSpawnExt; +use hashbrown::HashMap; +use itertools::Itertools; +use orchid_base::async_once_cell::OnceCell; +use orchid_base::char_filter::char_filter_match; +use orchid_base::clone; +use orchid_base::error::{OrcErrv, OrcRes}; +use orchid_base::interner::Tok; +use orchid_base::parse::Comment; +use orchid_base::reqnot::{ReqNot, Requester}; +use orchid_base::tree::ttv_from_api; +use ordered_float::NotNan; +use substack::{Stackframe, Substack}; + +use crate::api; +use crate::ctx::Ctx; +use crate::extension::{Extension, WeakExtension}; +use crate::tree::{Member, ParsTokTree}; + +#[derive(destructure)] +struct SystemInstData { + ctx: Ctx, + ext: Extension, + decl_id: api::SysDeclId, + lex_filter: api::CharFilter, + id: api::SysId, + const_root: OnceCell>, + line_types: Vec>, +} +impl Drop for SystemInstData { + fn drop(&mut self) { self.ext.system_drop(self.id); } +} +impl fmt::Debug for SystemInstData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SystemInstData") + .field("decl_id", &self.decl_id) + .field("lex_filter", &self.lex_filter) + .field("id", &self.id) + .field("const_root", &self.const_root) + .field("line_types", &self.line_types) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub struct System(Rc); +impl System { + pub fn id(&self) -> api::SysId { self.0.id } + pub fn ext(&self) -> &Extension { &self.0.ext } + pub fn ctx(&self) -> &Ctx { &self.0.ctx } + pub(crate) fn reqnot(&self) -> &ReqNot { &self.0.ext.reqnot() } + pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind { + self.reqnot().request(api::GetMember(self.0.id, id)).await + } + pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() } + pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) } + /// Have this system lex a part of the source. It is assumed that + /// [Self::can_lex] was called and returned true. + pub async fn lex>>( + &self, + source: Tok, + pos: u32, + r: impl FnMut(u32) -> F, + ) -> api::OrcResult> { + self.0.ext.lex_req(source, pos, self.id(), r).await + } + pub fn can_parse(&self, ltyp: Tok) -> bool { self.0.line_types.contains(<yp) } + pub fn line_types(&self) -> impl Iterator> + '_ { self.0.line_types.iter() } + pub async fn parse( + &self, + line: Vec, + exported: bool, + comments: Vec, + ) -> OrcRes> { + let line = + join_all(line.iter().map(|t| async { t.to_api(&mut |n, _| match *n {}).await })).await; + let comments = comments.iter().map(Comment::to_api).collect_vec(); + match self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line }).await { + Ok(parsed) => Ok(ttv_from_api(parsed, &mut self.ctx().clone(), &self.ctx().i).await), + Err(e) => Err(OrcErrv::from_api(&e, &self.ctx().i).await), + } + } + pub async fn request(&self, req: Vec) -> Vec { + self.reqnot().request(api::SysFwded(self.id(), req)).await + } + pub(crate) fn drop_atom(&self, drop: api::AtomId) { + let this = self.0.clone(); + (self.0.ctx.spawn.spawn_local(async move { + this.ctx.owned_atoms.write().await.remove(&drop); + })) + .expect("Failed to drop atom"); + } + pub async fn print(&self) -> String { + let ctor = (self.0.ext.system_ctors().find(|c| c.id() == self.0.decl_id)) + .expect("System instance with no associated constructor"); + format!("System({} @ {} #{})", ctor.name(), ctor.priority(), self.0.id.0) + } + pub fn downgrade(&self) -> WeakSystem { WeakSystem(Rc::downgrade(&self.0)) } +} + +pub struct WeakSystem(Weak); +impl WeakSystem { + pub fn upgrade(&self) -> Option { self.0.upgrade().map(System) } +} + +pub struct SystemCtor { + pub(crate) decl: api::SystemDecl, + pub(crate) ext: WeakExtension, +} +impl SystemCtor { + pub fn name(&self) -> &str { &self.decl.name } + pub fn priority(&self) -> NotNan { self.decl.priority } + pub fn depends(&self) -> impl ExactSizeIterator { + self.decl.depends.iter().map(|s| &**s) + } + pub fn id(&self) -> api::SysDeclId { self.decl.id } + pub async fn run<'a>(&self, depends: impl IntoIterator) -> System { + let depends = depends.into_iter().map(|si| si.id()).collect_vec(); + debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided"); + let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension"); + let id = ext.ctx().next_sys_id(); + let sys_inst = ext.reqnot().request(api::NewSystem { depends, id, system: self.decl.id }).await; + let data = System(Rc::new(SystemInstData { + decl_id: self.decl.id, + ext: ext.clone(), + ctx: ext.ctx().clone(), + lex_filter: sys_inst.lex_filter, + const_root: OnceCell::new(), + line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i))) + .await, + id, + })); + (data.0.const_root.get_or_init( + clone!(data, ext; stream! { + for (k, v) in sys_inst.const_root { + yield Member::from_api( + api::Member { name: k, kind: v }, + &mut vec![Tok::from_api(k, &ext.ctx().i).await], + &data, + ).await + } + }) + .collect(), + )) + .await; + ext.ctx().systems.write().await.insert(id, data.downgrade()); + data + } +} + +#[derive(Debug, Clone)] +pub enum SysResolvErr { + Loop(Vec), + Missing(String), +} + +pub async fn init_systems( + tgts: &[String], + exts: &[Extension], +) -> Result, SysResolvErr> { + let mut to_load = HashMap::<&str, &SystemCtor>::new(); + let mut to_find = tgts.iter().map(|s| s.as_str()).collect::>(); + while let Some(target) = to_find.pop_front() { + if to_load.contains_key(target) { + continue; + } + let ctor = (exts.iter()) + .flat_map(|e| e.system_ctors().filter(|c| c.name() == target)) + .max_by_key(|c| c.priority()) + .ok_or_else(|| SysResolvErr::Missing(target.to_string()))?; + to_load.insert(target, ctor); + to_find.extend(ctor.depends()); + } + let mut to_load_ordered = Vec::new(); + fn walk_deps<'a>( + graph: &mut HashMap<&str, &'a SystemCtor>, + list: &mut Vec<&'a SystemCtor>, + chain: Stackframe<&str>, + ) -> Result<(), SysResolvErr> { + if let Some(ctor) = graph.remove(chain.item) { + // if the above is none, the system is already queued. Missing systems are + // detected above + for dep in ctor.depends() { + if Substack::Frame(chain).iter().any(|c| *c == dep) { + let mut circle = vec![dep.to_string()]; + circle.extend(Substack::Frame(chain).iter().map(|s| s.to_string())); + return Err(SysResolvErr::Loop(circle)); + } + walk_deps(graph, list, Substack::Frame(chain).new_frame(dep))? + } + list.push(ctor); + } + Ok(()) + } + for tgt in tgts { + walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?; + } + let mut systems = HashMap::<&str, System>::new(); + for ctor in to_load_ordered.iter() { + let sys = ctor.run(ctor.depends().map(|n| &systems[n])).await; + systems.insert(ctor.name(), sys); + } + Ok(systems.into_values().collect_vec()) +} diff --git a/orchid-host/src/tree.rs b/orchid-host/src/tree.rs index b7a6dda..57c5e61 100644 --- a/orchid-host/src/tree.rs +++ b/orchid-host/src/tree.rs @@ -1,22 +1,25 @@ use std::fmt::Debug; use std::sync::{Mutex, OnceLock}; +use async_stream::stream; +use futures::{FutureExt, StreamExt}; use itertools::Itertools; use never::Never; +use orchid_base::clone; use orchid_base::error::OrcRes; -use orchid_base::interner::{Tok, intern}; +use orchid_base::interner::Tok; use orchid_base::location::Pos; use orchid_base::macros::mtreev_from_api; use orchid_base::name::Sym; use orchid_base::parse::{Comment, Import}; -use orchid_base::tree::{TokTree, Token}; +use orchid_base::tree::{AtomRepr, TokTree, Token}; use ordered_float::NotNan; -use substack::{Substack, with_iter_stack}; use crate::api; +use crate::atom::AtomHand; use crate::expr::Expr; -use crate::extension::{AtomHand, System}; use crate::macros::{MacTok, MacTree}; +use crate::system::System; pub type ParsTokTree = TokTree<'static, AtomHand, Never>; pub type ParsTok = Token<'static, AtomHand, Never>; @@ -37,54 +40,79 @@ pub enum ItemKind { } impl Item { - pub fn from_api(tree: api::Item, path: Substack>, sys: &System) -> Self { + pub async fn from_api(tree: api::Item, path: &mut Vec>, sys: &System) -> Self { let kind = match tree.kind { - api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys)), - api::ItemKind::Import(i) => - ItemKind::Import(Import { path: Sym::from_api(i).iter().collect(), name: None }), - api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e)), - api::ItemKind::Macro(api::MacroBlock { priority, rules }) => ItemKind::Macro(priority, { - Vec::from_iter(rules.into_iter().map(|api| Rule { - pos: Pos::from_api(&api.location), - pattern: mtreev_from_api(&api.pattern, &mut |a| { - MacTok::Atom(AtomHand::from_api(a.clone())) - }), - kind: RuleKind::Remote(sys.clone(), api.id), - comments: api.comments.iter().map(Comment::from_api).collect_vec(), - })) + api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys).await), + api::ItemKind::Import(name) => ItemKind::Import(Import { + path: Sym::from_api(name, &sys.ctx().i).await.iter().collect(), + name: None, }), + api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &sys.ctx().i).await), + api::ItemKind::Macro(macro_block) => { + let mut rules = Vec::new(); + for rule in macro_block.rules { + let mut comments = Vec::new(); + for comment in rule.comments { + comments.push(Comment::from_api(&comment, &sys.ctx().i).await); + } + let pos = Pos::from_api(&rule.location, &sys.ctx().i).await; + let pattern = mtreev_from_api(&rule.pattern, &sys.ctx().i, &mut { + clone!(pos, sys); + move |a| { + clone!(pos, sys); + Box::pin(async move { + MacTok::Atom(AtomHand::from_api(a, pos.clone(), &mut sys.ctx().clone()).await) + }) + } + }) + .await; + rules.push(Rule { pos, pattern, kind: RuleKind::Remote(sys.clone(), rule.id), comments }); + } + ItemKind::Macro(macro_block.priority, rules) + }, }; - let comments = tree.comments.iter().map(Comment::from_api).collect_vec(); - Self { pos: Pos::from_api(&tree.location), comments, kind } + let mut comments = Vec::new(); + for comment in tree.comments.iter() { + comments.push(Comment::from_api(comment, &sys.ctx().i).await) + } + Self { pos: Pos::from_api(&tree.location, &sys.ctx().i).await, comments, kind } } } -#[derive(Debug)] pub struct Member { pub name: Tok, pub kind: OnceLock, pub lazy: Mutex>, } impl Member { - pub fn from_api(api: api::Member, path: Substack>, sys: &System) -> Self { - let name = Tok::from_api(api.name); - let full_path = path.push(name.clone()); + pub async fn from_api(api: api::Member, path: &mut Vec>, sys: &System) -> Self { + path.push(Tok::from_api(api.name, &sys.ctx().i).await); let kind = match api.kind { api::MemberKind::Lazy(id) => { - return LazyMemberHandle(id, sys.clone(), intern(&full_path.unreverse())).into_member(name); + let handle = LazyMemberHandle(id, sys.clone(), path.clone()); + return handle.into_member(path.pop().unwrap()); }, api::MemberKind::Const(c) => MemberKind::Const(Code::from_expr( - CodeLocator::to_const(full_path.unreverse()), - Expr::from_api(&c, &mut ()), + CodeLocator::to_const(sys.ctx().i.i(&*path).await), + Expr::from_api(&c, &mut sys.ext().clone()).await, )), - api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, full_path, sys)), + api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, path, sys).await), }; + let name = path.pop().unwrap(); Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() } } pub fn new(name: Tok, kind: MemberKind) -> Self { Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() } } } +impl Debug for Member { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Member") + .field("name", &self.name) + .field("kind", &self.kind) + .finish_non_exhaustive() + } +} #[derive(Debug)] pub enum MemberKind { @@ -109,30 +137,27 @@ impl Module { .collect_vec(); Self { imports: vec![], exports, items } } - pub fn from_api(m: api::Module, path: Substack>, sys: &System) -> Self { - let mut output = Vec::new(); - for item in m.items.into_iter() { - let next = Item::from_api(item, path.clone(), sys); - output.push(next); - } - Self::new(output) + pub async fn from_api(m: api::Module, path: &mut Vec>, sys: &System) -> Self { + Self::new( + stream! { for item in m.items { yield Item::from_api(item, path, sys).boxed_local().await } } + .collect::>() + .await, + ) } } -#[derive(Debug)] -pub struct LazyMemberHandle(api::TreeId, System, Tok>>); +pub struct LazyMemberHandle(api::TreeId, System, Vec>); impl LazyMemberHandle { - pub fn run(self) -> OrcRes { - match self.1.get_tree(self.0) { + pub async fn run(self) -> OrcRes { + match self.1.get_tree(self.0).await { api::MemberKind::Const(c) => Ok(MemberKind::Const(Code { - bytecode: Expr::from_api(&c, &mut ()).into(), - locator: CodeLocator { steps: self.2, rule_loc: None }, + bytecode: Expr::from_api(&c, &mut self.1.ext().clone()).await.into(), + locator: CodeLocator { steps: self.1.ctx().i.i(&self.2).await, rule_loc: None }, source: None, })), - api::MemberKind::Module(m) => with_iter_stack(self.2.iter().cloned(), |path| { - Ok(MemberKind::Mod(Module::from_api(m, path, &self.1))) - }), - api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run(), + api::MemberKind::Module(m) => + Ok(MemberKind::Mod(Module::from_api(m, &mut { self.2 }, &self.1).await)), + api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run().boxed_local().await, } } pub fn into_member(self, name: Tok) -> Member { @@ -181,10 +206,8 @@ pub struct CodeLocator { rule_loc: Option<(u16, u16)>, } impl CodeLocator { - pub fn to_const(path: impl IntoIterator>) -> Self { - Self { steps: intern(&path.into_iter().collect_vec()), rule_loc: None } - } - pub fn to_rule(path: impl IntoIterator>, macro_i: u16, rule_i: u16) -> Self { - Self { steps: intern(&path.into_iter().collect_vec()), rule_loc: Some((macro_i, rule_i)) } + pub fn to_const(steps: Tok>>) -> Self { Self { steps, rule_loc: None } } + pub fn to_rule(steps: Tok>>, macro_i: u16, rule_i: u16) -> Self { + Self { steps, rule_loc: Some((macro_i, rule_i)) } } }