diff --git a/Cargo.lock b/Cargo.lock index fd505a1..9483484 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "ahash" version = "0.7.8" @@ -77,7 +92,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -87,7 +102,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -163,7 +178,7 @@ dependencies = [ "rustix", "slab", "tracing", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -217,7 +232,7 @@ dependencies = [ "rustix", "signal-hook-registry", "slab", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -286,6 +301,21 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + [[package]] name = "bitflags" version = "2.6.0" @@ -583,7 +613,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -760,6 +790,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "gloo-timers" version = "0.3.0" @@ -905,6 +941,16 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.22" @@ -938,6 +984,26 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.52.0", +] + [[package]] name = "never" version = "0.1.0" @@ -953,6 +1019,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -1000,6 +1075,7 @@ version = "0.1.0" dependencies = [ "async-once-cell", "async-std", + "async-stream", "derive_destructure", "dyn-clone", "futures", @@ -1076,6 +1152,9 @@ dependencies = [ name = "orchid-std" version = "0.1.0" dependencies = [ + "async-once-cell", + "async-std", + "futures", "itertools", "never", "once_cell", @@ -1085,17 +1164,22 @@ dependencies = [ "orchid-base", "orchid-extension", "ordered-float", + "tokio", ] [[package]] name = "orcx" version = "0.1.0" dependencies = [ + "async-stream", "camino", "clap", + "futures", "itertools", "orchid-base", "orchid-host", + "substack", + "tokio", ] [[package]] @@ -1113,6 +1197,29 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "paste" version = "1.0.15" @@ -1154,7 +1261,7 @@ dependencies = [ "pin-project-lite", "rustix", "tracing", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1273,6 +1380,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.11.1" @@ -1390,6 +1506,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "rustix" version = "0.38.43" @@ -1400,7 +1522,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1418,6 +1540,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "seahash" version = "4.1.0" @@ -1491,6 +1619,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "some_executor" version = "0.3.0" @@ -1611,6 +1755,35 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2 1.0.92", + "quote 1.0.38", + "syn 2.0.95", +] + [[package]] name = "toml_datetime" version = "0.6.8" @@ -1833,7 +2006,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", ] [[package]] diff --git a/orchid-api-derive/src/decode.rs b/orchid-api-derive/src/decode.rs index 04f19af..9e16e14 100644 --- a/orchid-api-derive/src/decode.rs +++ b/orchid-api-derive/src/decode.rs @@ -12,7 +12,9 @@ pub fn derive(input: TokenStream) -> TokenStream { let decode = decode_body(&input.data); let expanded = quote! { impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause { - async fn decode(mut read: std::pin::Pin<&mut R>) -> Self { + async fn decode( + mut read: std::pin::Pin<&mut R> + ) -> Self { #decode } } @@ -27,7 +29,8 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream { let exprs = fields.iter().map(|f| { let syn::Field { ty, ident, .. } = &f; quote! { - #ident : < #ty as orchid_api_traits::Decode>::decode(read.as_mut()).await + #ident : (Box::pin(< #ty as orchid_api_traits::Decode>::decode(read.as_mut())) + as std::pin::Pin>>).await } }); quote! { { #( #exprs, )* } } @@ -36,7 +39,8 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream { let exprs = fields.iter().map(|field| { let ty = &field.ty; quote! { - < #ty as orchid_api_traits::Decode>::decode(read.as_mut()).await, + (Box::pin(< #ty as orchid_api_traits::Decode>::decode(read.as_mut())) + as std::pin::Pin>>).await, } }); quote! { ( #( #exprs )* ) } diff --git a/orchid-api-derive/src/encode.rs b/orchid-api-derive/src/encode.rs index cf3d82d..7236b42 100644 --- a/orchid-api-derive/src/encode.rs +++ b/orchid-api-derive/src/encode.rs @@ -14,7 +14,12 @@ pub fn derive(input: TokenStream) -> TokenStream { let encode = encode_body(&input.data); let expanded = quote! { impl #e_impl_generics orchid_api_traits::Encode for #name #e_ty_generics #e_where_clause { - async fn encode(&self, mut write: std::pin::Pin<&mut W>) { #encode } + async fn encode( + &self, + mut write: std::pin::Pin<&mut W> + ) { + #encode + } } }; TokenStream::from(expanded) @@ -37,7 +42,8 @@ fn encode_body(data: &syn::Data) -> Option { let body = encode_items(&v.fields); quote! { Self::#ident #dest => { - (#i as u8).encode(write.as_mut()).await; + (Box::pin((#i as u8).encode(write.as_mut())) + as std::pin::Pin>>).await; #body } } @@ -53,7 +59,10 @@ fn encode_body(data: &syn::Data) -> Option { } fn encode_names(names: impl Iterator) -> pm2::TokenStream { - quote! { #( #names .encode(write.as_mut()).await; )* } + quote! { #( + (Box::pin(#names .encode(write.as_mut())) + as std::pin::Pin>>).await; + )* } } fn encode_items(fields: &syn::Fields) -> Option { diff --git a/orchid-api-traits/src/lib.rs b/orchid-api-traits/src/lib.rs index 742bc72..cbbcfa2 100644 --- a/orchid-api-traits/src/lib.rs +++ b/orchid-api-traits/src/lib.rs @@ -3,6 +3,7 @@ mod helpers; mod hierarchy; mod relations; +pub use async_std; pub use coding::*; pub use helpers::*; pub use hierarchy::*; diff --git a/orchid-api-traits/src/relations.rs b/orchid-api-traits/src/relations.rs index 4caf5c4..41afb70 100644 --- a/orchid-api-traits/src/relations.rs +++ b/orchid-api-traits/src/relations.rs @@ -1,10 +1,11 @@ +use core::fmt; use std::future::Future; use super::coding::Coding; use crate::helpers::enc_vec; -pub trait Request: Coding + Sized + Send + 'static { - type Response: Coding + Send + 'static; +pub trait Request: fmt::Debug + Coding + Sized + 'static { + type Response: fmt::Debug + Coding + 'static; } pub async fn respond(_: &R, rep: R::Response) -> Vec { enc_vec(&rep).await } @@ -16,11 +17,11 @@ pub async fn respond_with>( } pub trait Channel: 'static { - type Req: Coding + Sized + Send + 'static; - type Notif: Coding + Sized + Send + 'static; + type Req: Coding + Sized + 'static; + type Notif: Coding + Sized + 'static; } -pub trait MsgSet: Send + Sync + 'static { +pub trait MsgSet: Sync + 'static { type In: Channel; type Out: Channel; } diff --git a/orchid-api/src/logging.rs b/orchid-api/src/logging.rs index e665fb8..d46d1e6 100644 --- a/orchid-api/src/logging.rs +++ b/orchid-api/src/logging.rs @@ -6,6 +6,7 @@ use crate::ExtHostNotif; pub enum LogStrategy { StdErr, File(String), + Discard, } #[derive(Clone, Debug, Coding, Hierarchy)] diff --git a/orchid-api/src/proto.rs b/orchid-api/src/proto.rs index 4b3361a..b58dd5f 100644 --- a/orchid-api/src/proto.rs +++ b/orchid-api/src/proto.rs @@ -73,7 +73,7 @@ impl Request for Ping { } /// Requests running from the extension to the host -#[derive(Clone, Coding, Hierarchy)] +#[derive(Clone, Debug, Coding, Hierarchy)] #[extendable] pub enum ExtHostReq { Ping(Ping), diff --git a/orchid-base/Cargo.toml b/orchid-base/Cargo.toml index 9bce227..a6cb15a 100644 --- a/orchid-base/Cargo.toml +++ b/orchid-base/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] async-once-cell = "0.5.4" async-std = "1.13.0" +async-stream = "0.3.6" derive_destructure = "1.0.0" dyn-clone = "1.0.17" futures = "0.3.31" diff --git a/orchid-base/src/builtin.rs b/orchid-base/src/builtin.rs index 5f2116f..d441a70 100644 --- a/orchid-base/src/builtin.rs +++ b/orchid-base/src/builtin.rs @@ -1,9 +1,13 @@ use std::ops::Deref; +use std::rc::Rc; use futures::future::LocalBoxFuture; use crate::api; +pub type Spawner = Rc)>; +pub type RecvCB<'a> = Box FnOnce(&'b [u8]) -> LocalBoxFuture<'b, ()> + 'a>; + /// The 3 primary contact points with an extension are /// - send a message /// - wait for a message to arrive @@ -12,10 +16,7 @@ use crate::api; /// There are no ordering guarantees about these pub trait ExtPort { fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>; - fn recv<'a>( - &'a self, - cb: Box LocalBoxFuture<'_, ()> + 'a>, - ) -> LocalBoxFuture<'a, ()>; + fn recv<'a>(&'a self, cb: RecvCB<'a>) -> LocalBoxFuture<'a, ()>; } pub struct ExtInit { @@ -24,12 +25,7 @@ pub struct ExtInit { } impl ExtInit { pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await } - pub async fn recv<'a, 's: 'a>( - &'s self, - cb: Box LocalBoxFuture<'_, ()> + 'a>, - ) { - self.port.recv(Box::new(cb)).await - } + pub async fn recv<'a, 's: 'a>(&'s self, cb: RecvCB<'a>) { self.port.recv(Box::new(cb)).await } } impl Deref for ExtInit { type Target = api::ExtensionHeader; diff --git a/orchid-base/src/error.rs b/orchid-base/src/error.rs index d067e39..3810c7a 100644 --- a/orchid-base/src/error.rs +++ b/orchid-base/src/error.rs @@ -1,3 +1,4 @@ +use std::cell::RefCell; use std::fmt; use std::ops::Add; use std::sync::Arc; @@ -171,3 +172,17 @@ pub fn mk_errv( pub trait Reporter { fn report(&self, e: impl Into); } + +pub struct ReporterImpl { + errors: RefCell>, +} +impl ReporterImpl { + pub fn new() -> Self { Self { errors: RefCell::new(vec![]) } } + pub fn errv(self) -> Option { OrcErrv::new(self.errors.into_inner()).ok() } +} +impl Reporter for ReporterImpl { + fn report(&self, e: impl Into) { self.errors.borrow_mut().extend(e.into()) } +} +impl Default for ReporterImpl { + fn default() -> Self { Self::new() } +} diff --git a/orchid-base/src/interner.rs b/orchid-base/src/interner.rs index 8d1995d..c0c4afa 100644 --- a/orchid-base/src/interner.rs +++ b/orchid-base/src/interner.rs @@ -229,8 +229,6 @@ impl Interner { /// Intern some data; query its identifier if not known locally pub async fn i(&self, t: &(impl Internable + ?Sized)) -> Tok { let data = t.get_owned(); - let job = format!("{t:?} in {}", if self.master.is_some() { "replica" } else { "master" }); - eprintln!("Interning {job}"); let mut g = self.interners.lock().await; let typed = T::bimap(&mut g); if let Some(tok) = typed.by_value(&data) { @@ -243,7 +241,6 @@ impl Interner { }; let tok = Tok::new(data, marker); T::bimap(&mut g).insert(tok.clone()); - eprintln!("Interned {job}"); tok } /// Extern an identifier; query the data it represents if not known locally diff --git a/orchid-base/src/logging.rs b/orchid-base/src/logging.rs index 3828c0f..1c57dc2 100644 --- a/orchid-base/src/logging.rs +++ b/orchid-base/src/logging.rs @@ -20,9 +20,11 @@ impl Logger { } pub fn write_fmt(&self, fmt: Arguments) { match &self.0 { + api::LogStrategy::Discard => (), api::LogStrategy::StdErr => stderr().write_fmt(fmt).expect("Could not write to stderr!"), api::LogStrategy::File(f) => { - let mut file = File::open(f).expect("Could not open logfile"); + let mut file = (File::options().write(true).create(true).truncate(true).open(f)) + .expect("Could not open logfile"); file.write_fmt(fmt).expect("Could not write to logfile"); }, } diff --git a/orchid-base/src/macros.rs b/orchid-base/src/macros.rs index fd6f521..8d524f8 100644 --- a/orchid-base/src/macros.rs +++ b/orchid-base/src/macros.rs @@ -2,8 +2,7 @@ use std::marker::PhantomData; use std::rc::Rc; use std::sync::Arc; -use async_std::stream; -use async_std::sync::Mutex; +use async_stream::stream; use futures::future::LocalBoxFuture; use futures::{FutureExt, StreamExt}; use never::Never; @@ -63,7 +62,7 @@ pub enum MTok<'a, A> { Ref(Arc>), /// Used in the matcher to skip previous macro output which can only go in /// vectorial placeholders - Done(Arc>), + Done(Rc>), } impl<'a, A> MTok<'a, A> { pub(crate) async fn from_api( @@ -99,17 +98,17 @@ impl<'a, A> MTok<'a, A> { } pub async fn mtreev_from_api<'a, 'b, A>( - api: impl IntoIterator, + apiv: impl IntoIterator, i: &Interner, - do_atom: &mut impl MacroAtomFromApi<'a, A>, + do_atom: &'b mut (impl MacroAtomFromApi<'a, A> + 'b), ) -> Vec> { - let do_atom_lk = Mutex::new(do_atom); - stream::from_iter(api) - .then(|api| async { - MTree::from_api(api, &mut *do_atom_lk.lock().await, i).boxed_local().await - }) - .collect() - .await + stream! { + for api in apiv { + yield MTree::from_api(api, do_atom, i).boxed_local().await + } + } + .collect() + .await } pub async fn mtreev_to_api<'a: 'b, 'b, A: 'b>( diff --git a/orchid-base/src/parse.rs b/orchid-base/src/parse.rs index 9427dbc..58ae81e 100644 --- a/orchid-base/src/parse.rs +++ b/orchid-base/src/parse.rs @@ -192,13 +192,13 @@ pub struct Parsed<'a, 'b, T, A: AtomRepr, X: ExtraTok> { pub type ParseRes<'a, 'b, T, A, X> = OrcRes>; pub async fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>( - ctx: &impl Reporter, + ctx: &(impl Reporter + ?Sized), tail: Snippet<'a, 'b, A, X>, ) -> ParseRes<'a, 'b, Vec<(Import, Pos)>, A, X> { let ret = rec(ctx, tail).await; #[allow(clippy::type_complexity)] // it's an internal function pub async fn rec<'a, 'b, A: AtomRepr, X: ExtraTok>( - ctx: &impl Reporter, + ctx: &(impl Reporter + ?Sized), tail: Snippet<'a, 'b, A, X>, ) -> ParseRes<'a, 'b, Vec<(Vec>, Option>, Pos)>, A, X> { let comma = tail.i(",").await; diff --git a/orchid-base/src/reqnot.rs b/orchid-base/src/reqnot.rs index 4f26985..6c7b0b2 100644 --- a/orchid-base/src/reqnot.rs +++ b/orchid-base/src/reqnot.rs @@ -18,6 +18,7 @@ use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request}; use trait_set::trait_set; use crate::clone; +use crate::logging::Logger; pub struct Receipt<'a>(PhantomData<&'a mut ()>); @@ -110,16 +111,24 @@ impl Deref for RawReply { fn deref(&self) -> &Self::Target { get_id(&self.0[..]).1 } } -pub struct ReqNot(Arc>>); +pub struct ReqNot(Arc>>, Logger); impl ReqNot { - pub fn new(send: impl SendFn, notif: impl NotifFn, req: impl ReqFn) -> Self { - Self(Arc::new(Mutex::new(ReqNotData { - id: 1, - send: Box::new(send), - notif: Box::new(notif), - req: Box::new(req), - responses: HashMap::new(), - }))) + pub fn new( + logger: Logger, + send: impl SendFn, + notif: impl NotifFn, + req: impl ReqFn, + ) -> Self { + Self( + Arc::new(Mutex::new(ReqNotData { + id: 1, + send: Box::new(send), + notif: Box::new(notif), + req: Box::new(req), + responses: HashMap::new(), + })), + logger, + ) } /// Can be called from a polling thread or dispatched in any other way @@ -133,7 +142,7 @@ impl ReqNot { notif_cb(notif_val, self.clone()).await } else if 0 < id.bitand(1 << 63) { let sender = g.responses.remove(&!id).expect("Received response for invalid message"); - sender.send(message.to_vec()).await.unwrap(); + sender.send(message.to_vec()).await.unwrap() } else { let message = ::Req::decode(Pin::new(&mut &payload[..])).await; let mut req_cb = clone_box(&*g.req); @@ -154,28 +163,34 @@ impl ReqNot { pub trait DynRequester { type Transfer; + fn logger(&self) -> &Logger; /// Encode and send a request, then receive the response buffer. fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply>; } -pub struct MappedRequester<'a, T: 'a>(Box LocalBoxFuture<'a, RawReply> + 'a>); +pub struct MappedRequester<'a, T: 'a>(Box LocalBoxFuture<'a, RawReply> + 'a>, Logger); impl<'a, T> MappedRequester<'a, T> { - fn new(req: U) -> Self + fn new(req: U, logger: Logger) -> Self where T: Into { let req_arc = Arc::new(req); - MappedRequester(Box::new(move |t| { - Box::pin(clone!(req_arc; async move { req_arc.raw_request(t.into()).await})) - })) + MappedRequester( + Box::new(move |t| { + Box::pin(clone!(req_arc; async move { req_arc.raw_request(t.into()).await})) + }), + logger, + ) } } impl DynRequester for MappedRequester<'_, T> { type Transfer = T; + fn logger(&self) -> &Logger { &self.1 } fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply> { self.0(data) } } impl DynRequester for ReqNot { type Transfer = ::Req; + fn logger(&self) -> &Logger { &self.1 } fn raw_request(&self, req: Self::Transfer) -> LocalBoxFuture<'_, RawReply> { Box::pin(async move { let mut g = self.0.lock().await; @@ -203,17 +218,21 @@ pub trait Requester: DynRequester { ) -> impl Future; fn map<'a, U: Into>(self) -> MappedRequester<'a, U> where Self: Sized + 'a { - MappedRequester::new(self) + let logger = self.logger().clone(); + MappedRequester::new(self, logger) } } impl Requester for This { async fn request>(&self, data: R) -> R::Response { - R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await + let req = format!("{data:?}"); + let rep = R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await; + writeln!(self.logger(), "Request {req} got response {rep:?}"); + rep } } impl Clone for ReqNot { - fn clone(&self) -> Self { Self(self.0.clone()) } + fn clone(&self) -> Self { Self(self.0.clone(), self.1.clone()) } } #[cfg(test)] @@ -223,12 +242,14 @@ mod test { use async_std::sync::Mutex; use futures::FutureExt; + use orchid_api::LogStrategy; use orchid_api_derive::Coding; use orchid_api_traits::{Channel, Request}; use test_executors::spin_on; use super::{MsgSet, ReqNot}; use crate::clone; + use crate::logging::Logger; use crate::reqnot::Requester as _; #[derive(Clone, Debug, Coding, PartialEq)] @@ -252,8 +273,10 @@ mod test { #[test] fn notification() { spin_on(async { + let logger = Logger::new(LogStrategy::StdErr); let received = Arc::new(Mutex::new(None)); let receiver = ReqNot::::new( + logger.clone(), |_, _| panic!("Should not send anything"), clone!(received; move |notif, _| clone!(received; async move { *received.lock().await = Some(notif); @@ -261,6 +284,7 @@ mod test { |_, _| panic!("Not receiving a request"), ); let sender = ReqNot::::new( + logger, clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move { receiver.receive(d).await }))), @@ -277,8 +301,10 @@ mod test { #[test] fn request() { spin_on(async { + let logger = Logger::new(LogStrategy::StdErr); let receiver = Rc::new(Mutex::>>::new(None)); let sender = Rc::new(ReqNot::::new( + logger.clone(), clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move { receiver.lock().await.as_ref().unwrap().receive(d).await }))), @@ -286,6 +312,7 @@ mod test { |_, _| panic!("Should not receive request"), )); *receiver.lock().await = Some(ReqNot::new( + logger, clone!(sender; move |d, _| clone!(sender; Box::pin(async move { sender.receive(d).await }))), diff --git a/orchid-base/src/tree.rs b/orchid-base/src/tree.rs index b69609c..48f83ed 100644 --- a/orchid-base/src/tree.rs +++ b/orchid-base/src/tree.rs @@ -7,8 +7,7 @@ use std::ops::Range; use std::sync::Arc; pub use api::PhKind; -use async_std::stream; -use async_std::sync::Mutex; +use async_stream::stream; use futures::future::{LocalBoxFuture, join_all}; use futures::{FutureExt, StreamExt}; use itertools::Itertools; @@ -161,14 +160,13 @@ pub async fn ttv_from_api( ctx: &mut A::Ctx, i: &Interner, ) -> Vec> { - let ctx_lk = Mutex::new(ctx); - stream::from_iter(tokv.into_iter()) - .then(|t| async { - let t = t; - TokTree::::from_api(t.borrow(), *ctx_lk.lock().await, i).boxed_local().await - }) - .collect() - .await + stream! { + for tok in tokv { + yield TokTree::::from_api(tok.borrow(), ctx, i).boxed_local().await + } + } + .collect() + .await } pub async fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>( @@ -186,11 +184,13 @@ pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>( tokv: impl IntoIterator>, do_extra: &mut impl FnMut(X, Range) -> api::TokenTree, ) -> Vec { - let mut new_tokv = Vec::new(); - for item in tokv { - new_tokv.push(item.into_api(do_extra).await) + stream! { + for tok in tokv { + yield tok.into_api(do_extra).await + } } - new_tokv + .collect() + .await } /// This takes a position and not a range because it assigns the range to diff --git a/orchid-extension/src/atom.rs b/orchid-extension/src/atom.rs index 3c39f55..b82328d 100644 --- a/orchid-extension/src/atom.rs +++ b/orchid-extension/src/atom.rs @@ -148,7 +148,7 @@ pub trait AtomMethod: Request { const NAME: &str; } pub trait Supports: AtomCard { - fn handle(&self, ctx: SysCtx, req: M) -> LocalBoxFuture<'_, ::Response>; + fn handle(&self, ctx: SysCtx, req: M) -> impl Future::Response>; } trait_set! { diff --git a/orchid-extension/src/atom_owned.rs b/orchid-extension/src/atom_owned.rs index cc3da20..7adf00f 100644 --- a/orchid-extension/src/atom_owned.rs +++ b/orchid-extension/src/atom_owned.rs @@ -144,8 +144,8 @@ impl AtomDynfo for OwnedAtomDynfo { pub trait DeserializeCtx: Sized { fn read(&mut self) -> impl Future; fn is_empty(&self) -> bool; - fn assert_empty(self) { assert!(self.is_empty(), "Bytes found after decoding") } - fn decode(mut self) -> impl Future { + fn assert_empty(&self) { assert!(self.is_empty(), "Bytes found after decoding") } + fn decode(&mut self) -> impl Future { async { let t = self.read().await; self.assert_empty(); diff --git a/orchid-extension/src/entrypoint.rs b/orchid-extension/src/entrypoint.rs index 7553975..725e632 100644 --- a/orchid-extension/src/entrypoint.rs +++ b/orchid-extension/src/entrypoint.rs @@ -18,7 +18,7 @@ use hashbrown::HashMap; use itertools::Itertools; use orchid_api::ApplyMacro; use orchid_api_traits::{Decode, Encode, enc_vec}; -use orchid_base::builtin::ExtPort; +use orchid_base::builtin::{ExtPort, Spawner}; use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter}; use orchid_base::clone; use orchid_base::interner::{Interner, Tok}; @@ -127,7 +127,7 @@ impl ExtPort for ExtensionOwner { } } -async fn extension_main_logic(data: ExtensionData, spawner: Rc) { +pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) { let api::HostHeader { log_strategy } = api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await; let mut buf = Vec::new(); @@ -142,7 +142,7 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) std::io::stdout().write_all(&buf).unwrap(); std::io::stdout().flush().unwrap(); let exiting = Arc::new(AtomicBool::new(false)); - let logger = Arc::new(Logger::new(log_strategy)); + let logger = Logger::new(log_strategy); let interner_cell = Rc::new(RefCell::new(None::>)); let interner_weak = Rc::downgrade(&interner_cell); let obj_store = ObjStore::default(); @@ -159,10 +159,8 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) }.boxed_local()) }); let rn = ReqNot::::new( - clone!(logger; move |a, _| clone!(logger; async move { - logger.log_buf("Upsending", a); - send_parent_msg(a).await.unwrap() - }.boxed_local())), + logger.clone(), + move |a, _| async move { send_parent_msg(a).await.unwrap() }.boxed_local(), clone!(systems, exiting, mk_ctx, obj_store; move |n, reqnot| { clone!(systems, exiting, mk_ctx, obj_store; async move { match n { @@ -209,7 +207,7 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) .then(|(k, v)| { let (req, lazy_mems, rules) = (&hand, &lazy_mems, &rules); clone!(i, ctx; async move { - let name = i.i::(&k).await.to_api(); + let name = i.i(&k).await.to_api(); let value = v.into_api(&mut TIACtxImpl { lazy_members: &mut *lazy_mems.lock().await, rules: &mut *rules.lock().await, @@ -434,6 +432,6 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc) *interner_cell.borrow_mut() = Some(Rc::new(Interner::new_replica(rn.clone().map()))); while !exiting.load(Ordering::Relaxed) { let rcvd = recv_parent_msg().await.unwrap(); - rn.receive(&rcvd).await + spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd).await }))) } } diff --git a/orchid-extension/src/expr.rs b/orchid-extension/src/expr.rs index 6205785..9f4c507 100644 --- a/orchid-extension/src/expr.rs +++ b/orchid-extension/src/expr.rs @@ -3,7 +3,6 @@ use std::rc::Rc; use async_once_cell::OnceCell; use derive_destructure::destructure; -use futures::task::LocalSpawnExt; use orchid_base::error::OrcErrv; use orchid_base::location::Pos; use orchid_base::reqnot::Requester; @@ -34,10 +33,8 @@ impl fmt::Debug for ExprHandle { impl Drop for ExprHandle { fn drop(&mut self) { let notif = api::Release(self.ctx.id, self.tk); - let SysCtx { reqnot, spawner, logger, .. } = self.ctx.clone(); - if let Err(e) = spawner.spawn_local(async move { reqnot.notify(notif).await }) { - writeln!(logger, "Failed to schedule notification about resource release: {e}"); - } + let SysCtx { reqnot, spawner, .. } = self.ctx.clone(); + spawner(Box::pin(async move { reqnot.notify(notif).await })) } } diff --git a/orchid-extension/src/func_atom.rs b/orchid-extension/src/func_atom.rs index 0e8a0cf..8601816 100644 --- a/orchid-extension/src/func_atom.rs +++ b/orchid-extension/src/func_atom.rs @@ -84,7 +84,7 @@ impl OwnedAtom for Fun { self.path.to_api().encode(write).await; self.args.clone() } - async fn deserialize(ctx: impl DeserializeCtx, args: Self::Refs) -> Self { + async fn deserialize(mut ctx: impl DeserializeCtx, args: Self::Refs) -> Self { let sys = ctx.sys(); let path = Sym::from_api(ctx.decode().await, &sys.i).await; let (arity, fun) = FUNS.with(|f| f.clone()).lock().await.get(&path).unwrap().clone(); @@ -128,6 +128,8 @@ impl OwnedAtom for Lambda { } mod expr_func_derives { + use std::future::Future; + use orchid_base::error::OrcRes; use super::ExprFunc; @@ -140,14 +142,14 @@ mod expr_func_derives { paste::paste!{ impl< $($t: TryFromExpr, )* - Out: ToExpr, - Func: Fn($($t,)*) -> Out + Clone + Send + Sync + 'static - > ExprFunc<($($t,)*), Out> for Func { + Fut: Future, + Func: Fn($($t,)*) -> Fut + Clone + Send + Sync + 'static + > ExprFunc<($($t,)*), Fut::Output> for Func { const ARITY: u8 = $arity; async fn apply(&self, v: Vec) -> OrcRes { assert_eq!(v.len(), Self::ARITY.into(), "Arity mismatch"); let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above")); - Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).to_expr()) + Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_expr()) } } } diff --git a/orchid-extension/src/gen_expr.rs b/orchid-extension/src/gen_expr.rs index ab3d8f4..3bd70e9 100644 --- a/orchid-extension/src/gen_expr.rs +++ b/orchid-extension/src/gen_expr.rs @@ -1,3 +1,5 @@ +use std::future::Future; + use futures::FutureExt; use orchid_base::error::{OrcErr, OrcErrv}; use orchid_base::location::Pos; @@ -98,9 +100,9 @@ pub fn bot(ev: impl IntoIterator) -> GExpr { inherit(GExprKind::Bottom(OrcErrv::new(ev).unwrap())) } -pub fn with( +pub fn with>( expr: GExpr, - cont: impl Fn(I) -> O + Clone + Send + Sync + 'static, + cont: impl Fn(I) -> Fut + Clone + Send + Sync + 'static, ) -> GExpr { call([lambda(0, [seq([arg(0), call([Lambda::new(cont).to_expr(), arg(0)])])]), expr]) } diff --git a/orchid-extension/src/msg.rs b/orchid-extension/src/msg.rs index 07dc633..07da0ac 100644 --- a/orchid-extension/src/msg.rs +++ b/orchid-extension/src/msg.rs @@ -1,9 +1,15 @@ use std::pin::pin; -use async_std::io; +use async_once_cell::OnceCell; +use async_std::io::{self, Stdout}; +use async_std::sync::Mutex; use orchid_base::msg::{recv_msg, send_msg}; +static STDOUT: OnceCell> = OnceCell::new(); + pub async fn send_parent_msg(msg: &[u8]) -> io::Result<()> { - send_msg(pin!(io::stdout()), msg).await + let stdout_lk = STDOUT.get_or_init(async { Mutex::new(io::stdout()) }).await; + let mut stdout_g = stdout_lk.lock().await; + send_msg(pin!(&mut *stdout_g), msg).await } pub async fn recv_parent_msg() -> io::Result> { recv_msg(pin!(io::stdin())).await } diff --git a/orchid-extension/src/system.rs b/orchid-extension/src/system.rs index b183cf0..0303d9d 100644 --- a/orchid-extension/src/system.rs +++ b/orchid-extension/src/system.rs @@ -4,13 +4,12 @@ use std::future::Future; use std::num::NonZero; use std::pin::Pin; use std::rc::Rc; -use std::sync::Arc; use futures::future::LocalBoxFuture; -use futures::task::LocalSpawn; use hashbrown::HashMap; use orchid_api_traits::{Coding, Decode}; use orchid_base::boxed_iter::BoxedIter; +use orchid_base::builtin::Spawner; use orchid_base::interner::Interner; use orchid_base::logging::Logger; use orchid_base::reqnot::{Receipt, ReqNot}; @@ -133,10 +132,10 @@ where A: AtomicFeatures { #[derive(Clone)] pub struct SysCtx { pub reqnot: ReqNot, - pub spawner: Rc, + pub spawner: Spawner, pub id: api::SysId, pub cted: CtedObj, - pub logger: Arc, + pub logger: Logger, pub obj_store: ObjStore, pub i: Rc, } diff --git a/orchid-extension/src/tree.rs b/orchid-extension/src/tree.rs index 8ef7b53..87a59e1 100644 --- a/orchid-extension/src/tree.rs +++ b/orchid-extension/src/tree.rs @@ -71,16 +71,16 @@ impl GenItem { pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec { with_export(GenMember { name: name.to_string(), kind: MemKind::Const(value.to_expr()) }, public) } -pub async fn module( +pub fn module( public: bool, name: &str, imports: impl IntoIterator, items: impl IntoIterator>, ) -> Vec { - let (name, kind) = root_mod(name, imports, items).await; + let (name, kind) = root_mod(name, imports, items); with_export(GenMember { name, kind }, public) } -pub async fn root_mod( +pub fn root_mod( name: &str, imports: impl IntoIterator, items: impl IntoIterator>, @@ -91,7 +91,7 @@ pub async fn root_mod( }; (name.to_string(), kind) } -pub async fn fun(exported: bool, name: &str, xf: impl ExprFunc) -> Vec { +pub fn fun(exported: bool, name: &str, xf: impl ExprFunc) -> Vec { let fac = LazyMemberFactory::new(move |sym| async { MemKind::Const(Fun::new(sym, xf).await.to_expr()) }); with_export(GenMember { name: name.to_string(), kind: MemKind::Lazy(fac) }, exported) diff --git a/orchid-host/src/atom.rs b/orchid-host/src/atom.rs index cfbec68..68bb6f7 100644 --- a/orchid-host/src/atom.rs +++ b/orchid-host/src/atom.rs @@ -86,7 +86,7 @@ impl AtomHand { impl AtomRepr for AtomHand { type Ctx = Ctx; async fn from_api(atom: &orchid_api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self { - Self::new(atom.clone(), &ctx).await + Self::new(atom.clone(), ctx).await } async fn to_api(&self) -> orchid_api::Atom { self.api_ref() } async fn print(&self) -> String { self.to_string().await } diff --git a/orchid-host/src/ctx.rs b/orchid-host/src/ctx.rs index c9e697d..37f2fea 100644 --- a/orchid-host/src/ctx.rs +++ b/orchid-host/src/ctx.rs @@ -1,12 +1,12 @@ use std::cell::RefCell; -use std::num::NonZeroU16; +use std::num::{NonZero, NonZeroU16}; use std::rc::Rc; use std::{fmt, ops}; use async_std::sync::RwLock; -use futures::task::LocalSpawn; use hashbrown::HashMap; use orchid_api::SysId; +use orchid_base::builtin::Spawner; use orchid_base::interner::Interner; use crate::api; @@ -15,7 +15,7 @@ use crate::system::{System, WeakSystem}; pub struct CtxData { pub i: Rc, - pub spawn: Rc, + pub spawn: Spawner, pub systems: RwLock>, pub system_id: RefCell, pub owned_atoms: RwLock>, @@ -24,9 +24,18 @@ pub struct CtxData { pub struct Ctx(Rc); impl ops::Deref for Ctx { type Target = CtxData; - fn deref(&self) -> &Self::Target { &*self.0 } + fn deref(&self) -> &Self::Target { &self.0 } } impl Ctx { + pub fn new(spawn: Spawner) -> Self { + Self(Rc::new(CtxData { + spawn, + i: Rc::default(), + systems: RwLock::default(), + system_id: RefCell::new(NonZero::new(1).unwrap()), + owned_atoms: RwLock::default(), + })) + } pub(crate) async fn system_inst(&self, id: api::SysId) -> Option { self.systems.read().await.get(&id).and_then(WeakSystem::upgrade) } diff --git a/orchid-host/src/expr.rs b/orchid-host/src/expr.rs index db3705a..e680060 100644 --- a/orchid-host/src/expr.rs +++ b/orchid-host/src/expr.rs @@ -1,7 +1,6 @@ use std::collections::VecDeque; use std::num::NonZeroU64; use std::rc::{Rc, Weak}; -use std::sync::atomic::AtomicBool; use async_std::sync::RwLock; use futures::FutureExt; @@ -19,7 +18,6 @@ pub type ExprParseCtx = Extension; #[derive(Debug)] pub struct ExprData { - is_canonical: AtomicBool, pos: Pos, kind: RwLock, } @@ -51,7 +49,7 @@ impl Expr { } let pos = Pos::from_api(&api.location, &ctx.ctx().i).await; let kind = RwLock::new(ExprKind::from_api(&api.kind, pos.clone(), ctx).boxed_local().await); - Self(Rc::new(ExprData { is_canonical: AtomicBool::new(false), pos, kind })) + Self(Rc::new(ExprData { pos, kind })) } pub async fn to_api(&self) -> api::InspectedKind { use api::InspectedKind as K; diff --git a/orchid-host/src/extension.rs b/orchid-host/src/extension.rs index 15f09bd..c1725df 100644 --- a/orchid-host/src/extension.rs +++ b/orchid-host/src/extension.rs @@ -9,7 +9,6 @@ use async_std::sync::Mutex; use derive_destructure::destructure; use futures::FutureExt; use futures::future::{join, join_all}; -use futures::task::LocalSpawnExt; use hashbrown::HashMap; use itertools::Itertools; use orchid_api::HostMsgSet; @@ -18,7 +17,6 @@ use orchid_base::builtin::ExtInit; use orchid_base::clone; use orchid_base::interner::Tok; use orchid_base::logging::Logger; -use orchid_base::macros::mtreev_from_api; use orchid_base::reqnot::{ReqNot, Requester as _}; use orchid_base::tree::AtomRepr; @@ -26,7 +24,6 @@ use crate::api; use crate::atom::AtomHand; use crate::ctx::Ctx; use crate::expr_store::ExprStore; -use crate::macros::{macro_recur, macro_treev_to_api}; use crate::system::SystemCtor; pub struct ReqPair(R, Sender); @@ -45,9 +42,13 @@ pub struct ExtensionData { next_pars: RefCell, exprs: ExprStore, lex_recur: Mutex>>>, + mac_recur: Mutex>>>, } impl Drop for ExtensionData { - fn drop(&mut self) { self.reqnot.notify(api::HostExtNotif::Exit); } + fn drop(&mut self) { + let reqnot = self.reqnot.clone(); + (self.ctx.spawn)(Box::pin(async move { reqnot.notify(api::HostExtNotif::Exit).await })) + } } #[derive(Clone)] @@ -60,17 +61,19 @@ impl Extension { systems: (init.systems.iter().cloned()) .map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) }) .collect(), - logger, + logger: logger.clone(), init, next_pars: RefCell::new(NonZeroU64::new(1).unwrap()), lex_recur: Mutex::default(), + mac_recur: Mutex::default(), reqnot: ReqNot::new( + logger, clone!(weak; move |sfn, _| clone!(weak; async move { let data = weak.upgrade().unwrap(); - data.logger.log_buf("Downsending", sfn); data.init.send(sfn).await }.boxed_local())), - clone!(weak; move |notif, _| clone!(weak; async move { + clone!(weak; move |notif, _| { + clone!(weak; Box::pin(async move { let this = Extension(weak.upgrade().unwrap()); match notif { api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => { @@ -90,12 +93,12 @@ impl Extension { }, api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str), } - }.boxed_local())), + }))}), { clone!(weak, ctx); move |hand, req| { clone!(weak, ctx); - async move { + Box::pin(async move { let this = Self(weak.upgrade().unwrap()); let i = this.ctx().i.clone(); match req { @@ -124,10 +127,12 @@ impl Extension { hand.handle(fw, &sys.request(body.clone()).await).await }, api::ExtHostReq::SubLex(sl) => { - let (rep_in, rep_out) = channel::bounded(0); - let lex_g = this.0.lex_recur.lock().await; - let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid"); - req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap(); + let (rep_in, rep_out) = channel::bounded(1); + { + let lex_g = this.0.lex_recur.lock().await; + let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid"); + req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap(); + } hand.handle(&sl, &rep_out.recv().await.unwrap()).await }, api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => { @@ -140,19 +145,17 @@ impl Extension { }) .await }, - api::ExtHostReq::RunMacros(ref rm @ api::RunMacros { ref run_id, ref query }) => { - let mtreev = - mtreev_from_api(query, &i, &mut |_| panic!("Atom in macro recur")).await; - match macro_recur(*run_id, mtreev).await { - Some(x) => hand.handle(rm, &Some(macro_treev_to_api(*run_id, x).await)).await, - None => hand.handle(rm, &None).await, - } + api::ExtHostReq::RunMacros(rm) => { + let (rep_in, rep_out) = channel::bounded(1); + let lex_g = this.0.mac_recur.lock().await; + let req_in = lex_g.get(&rm.run_id).expect("Sublex for nonexistent lexid"); + req_in.send(ReqPair(rm.clone(), rep_in)).await.unwrap(); + hand.handle(&rm, &rep_out.recv().await.unwrap()).await }, api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => hand.handle(eap, &AtomHand::new(atom.clone(), &ctx).await.print().await).await, } - } - .boxed_local() + }) } }, ), @@ -185,7 +188,7 @@ impl Extension { // get unique lex ID let id = api::ParsId(self.next_pars()); // create and register channel - let (req_in, req_out) = channel::bounded(0); + let (req_in, req_out) = channel::bounded(1); self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released let (ret, ()) = join( async { @@ -207,19 +210,15 @@ impl Extension { } pub async fn recv_one(&self) { let reqnot = self.0.reqnot.clone(); - self - .0 - .init - .recv(Box::new(move |msg| async move { reqnot.receive(msg).await }.boxed_local())) + (self.0.init.recv(Box::new(move |msg| async move { reqnot.receive(msg).await }.boxed_local()))) .await; } pub fn system_drop(&self, id: api::SysId) { let rc = self.clone(); - (self.ctx().spawn.spawn_local(async move { + (self.ctx().spawn)(Box::pin(async move { rc.reqnot().notify(api::SystemDrop(id)).await; rc.ctx().systems.write().await.remove(&id); })) - .expect("Failed to drop system!"); } pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) } } diff --git a/orchid-host/src/lex.rs b/orchid-host/src/lex.rs index 5a5412d..37871a4 100644 --- a/orchid-host/src/lex.rs +++ b/orchid-host/src/lex.rs @@ -34,7 +34,7 @@ impl<'a> LexCtx<'a> { tail: &self.source[pos as usize..], systems: self.systems, sub_trees: &mut *self.sub_trees, - ctx: &self.ctx, + ctx: self.ctx, } } pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 } @@ -76,7 +76,7 @@ impl<'a> LexCtx<'a> { } } -pub async fn lex_once<'a>(ctx: &mut LexCtx<'a>) -> OrcRes { +pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes { let start = ctx.get_pos(); assert!( !ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space), @@ -140,7 +140,7 @@ pub async fn lex_once<'a>(ctx: &mut LexCtx<'a>) -> OrcRes { let numstr = ctx.get_start_matches(|x| x != ')').trim(); match parse_num(numstr) { Ok(num) => ParsTok::Macro(Some(num.to_f64())), - Err(e) => return Err(num_to_err(e, pos, &*ctx.ctx.i).await.into()), + Err(e) => return Err(num_to_err(e, pos, &ctx.ctx.i).await.into()), } } else { ParsTok::Macro(None) @@ -154,8 +154,9 @@ pub async fn lex_once<'a>(ctx: &mut LexCtx<'a>) -> OrcRes { let errors_lck = &Mutex::new(&mut errors); let lx = sys .lex(source, pos, |pos| async move { - match lex_once(&mut ctx_lck.lock().await.push(pos)).boxed_local().await { - Ok(t) => Some(api::SubLexed { pos, ticket: ctx_lck.lock().await.add_subtree(t) }), + let mut ctx_g = ctx_lck.lock().await; + match lex_once(&mut ctx_g.push(pos)).boxed_local().await { + Ok(t) => Some(api::SubLexed { pos, ticket: ctx_g.add_subtree(t) }), Err(e) => { errors_lck.lock().await.push(e); None @@ -166,9 +167,12 @@ pub async fn lex_once<'a>(ctx: &mut LexCtx<'a>) -> OrcRes { match lx { Err(e) => return Err( - errors.into_iter().fold(OrcErrv::from_api(&e, &*ctx.ctx.i).await, |a, b| a + b), + errors.into_iter().fold(OrcErrv::from_api(&e, &ctx.ctx.i).await, |a, b| a + b), ), - Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos)).await), + Ok(Some(lexed)) => { + ctx.set_pos(lexed.pos); + return Ok(tt_to_owned(&lexed.expr, ctx).await); + }, Ok(None) => match errors.into_iter().reduce(|a, b| a + b) { Some(errors) => return Err(errors), None => continue, @@ -196,13 +200,13 @@ async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree Atom(atom => AtomHand::from_api(atom, Pos::Range(api.range.clone()), &mut ctx.ctx.clone()).await ), - Bottom(err => OrcErrv::from_api(err, &*ctx.ctx.i).await), + Bottom(err => OrcErrv::from_api(err, &ctx.ctx.i).await), LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await), - Name(name => Tok::from_api(*name, &*ctx.ctx.i).await), + Name(name => Tok::from_api(*name, &ctx.ctx.i).await), S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await), BR, NS, Comment(c.clone()), - Ph(ph => Ph::from_api(ph, &*ctx.ctx.i).await), + Ph(ph => Ph::from_api(ph, &ctx.ctx.i).await), Macro(*prio), } { api::Token::Slot(id) => return ctx.rm_subtree(*id), @@ -216,7 +220,7 @@ async fn ttv_to_owned<'a>( ) -> Vec { let mut out = Vec::new(); for tt in api { - out.push(tt_to_owned(&tt, ctx).await) + out.push(tt_to_owned(tt, ctx).await) } out } diff --git a/orchid-host/src/macros.rs b/orchid-host/src/macros.rs index 3d27d4a..dff356c 100644 --- a/orchid-host/src/macros.rs +++ b/orchid-host/src/macros.rs @@ -1,17 +1,16 @@ use std::rc::Rc; -use async_std::sync::RwLock; use futures::FutureExt; use hashbrown::{HashMap, HashSet}; use itertools::Itertools; -use orchid_base::interner::Interner; +use orchid_base::clone; use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api}; use orchid_base::name::Sym; use trait_set::trait_set; use crate::api; use crate::atom::AtomHand; -use crate::rule::matcher::{NamedMatcher, PriodMatcher}; +use crate::ctx::Ctx; use crate::rule::state::MatchState; use crate::tree::Code; @@ -22,36 +21,27 @@ trait_set! { trait MacroCB = Fn(Vec) -> Option>; } -thread_local! { - static RECURSION: RwLock>> = RwLock::default(); - static MACRO_SLOTS: RwLock>>> = - RwLock::default(); -} +type Slots = HashMap>; -pub async fn macro_recur(run_id: api::ParsId, input: Vec) -> Option> { - (RECURSION.read().unwrap()[&run_id])(input) -} - -pub async fn macro_treev_to_api(run_id: api::ParsId, mtree: Vec) -> Vec { - let mut g = MACRO_SLOTS.write().unwrap(); - let run_cache = g.get_mut(&run_id).expect("Parser run not found"); +pub async fn macro_treev_to_api(mtree: Vec, slots: &mut Slots) -> Vec { mtreev_to_api(&mtree, &mut |a: &AtomHand| { - let id = api::MacroTreeId((run_cache.len() as u64 + 1).try_into().unwrap()); - run_cache.insert(id, Rc::new(MacTok::Atom(a.clone()))); - api::MacroToken::Slot(id) - }) -} - -pub async fn macro_treev_from_api(api: Vec, i: &Interner) -> Vec { - mtreev_from_api(&api, i, &mut |atom| { - async { MacTok::Atom(AtomHand::from_api(atom.clone())) }.boxed_local() + let id = api::MacroTreeId((slots.len() as u64 + 1).try_into().unwrap()); + slots.insert(id, Rc::new(MacTok::Atom(a.clone()))); + async move { api::MacroToken::Slot(id) }.boxed_local() }) .await } -pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option> { - let mut slots = (MACRO_SLOTS.write().unwrap()).remove(&run_id).expect("Run not found"); - return work(&mut slots, tree); +pub async fn macro_treev_from_api(api: Vec, ctx: Ctx) -> Vec { + mtreev_from_api(&api, &ctx.clone().i, &mut move |atom| { + clone!(ctx); + Box::pin(async move { MacTok::Atom(AtomHand::new(atom.clone(), &ctx).await) }) + }) + .await +} + +pub fn deslot_macro(tree: &[MacTree], slots: &mut Slots) -> Option> { + return work(slots, tree); fn work( slots: &mut HashMap>, tree: &[MacTree], @@ -90,74 +80,6 @@ pub struct Macro { cases: Vec<(Matcher, Code)>, } -pub struct MacroRepo { - named: HashMap>>, - prio: Vec>, -} -impl MacroRepo { - /// TODO: the recursion inside this function needs to be moved into Orchid. - /// See the markdown note - pub fn process_exprv(&self, target: &[MacTree], i: &Interner) -> Option> { - let mut workcp = target.to_vec(); - let mut lexicon; - - 'try_named: loop { - lexicon = HashSet::new(); - target.iter().for_each(|tgt| fill_lexicon(tgt, &mut lexicon)); - - for (idx, tree) in workcp.iter().enumerate() { - let MacTok::Name(name) = &*tree.tok else { continue }; - let matches = (self.named.get(name).into_iter().flatten()) - .filter(|m| m.deps.is_subset(&lexicon)) - .filter_map(|mac| { - (mac.cases.iter()) - .find_map(|cas| cas.0.apply(&workcp[idx..], i, |_| false).map(|s| (cas, s))) - }) - .collect_vec(); - assert!( - matches.len() < 2, - "Multiple conflicting matches on {:?}: {:?}", - &workcp[idx..], - matches - ); - let Some((case, (state, tail))) = matches.into_iter().next() else { continue }; - let inj = (run_body(&case.1, state).into_iter()) - .map(|MacTree { pos, tok }| MacTree { pos, tok: Rc::new(MacTok::Done(tok)) }); - workcp.splice(idx..(workcp.len() - tail.len()), inj); - continue 'try_named; - } - break; - } - - if let Some(((_, body), state)) = (self.prio.iter()) - .filter(|mac| mac.deps.is_subset(&lexicon)) - .flat_map(|mac| &mac.cases) - .find_map(|case| case.0.apply(&workcp, |_| false).map(|state| (case, state))) - { - return Some(run_body(body, state)); - } - - let results = (workcp.into_iter()) - .map(|mt| match &*mt.tok { - MTok::S(p, body) => self.process_exprv(body, i).map(|body| MTok::S(*p, body).at(mt.pos)), - MTok::Lambda(arg, body) => - match (self.process_exprv(arg, i), self.process_exprv(body, i)) { - (Some(arg), Some(body)) => Some(MTok::Lambda(arg, body).at(mt.pos)), - (Some(arg), None) => Some(MTok::Lambda(arg, body.to_vec()).at(mt.pos)), - (None, Some(body)) => Some(MTok::Lambda(arg.to_vec(), body).at(mt.pos)), - (None, None) => None, - }, - _ => None, - }) - .collect_vec(); - results.iter().any(Option::is_some).then(|| { - (results.into_iter().zip(target)) - .map(|(opt, fb)| opt.unwrap_or_else(|| fb.clone())) - .collect_vec() - }) - } -} - fn fill_lexicon(tgt: &MacTree, lexicon: &mut HashSet) { match &*tgt.tok { MTok::Name(n) => { diff --git a/orchid-host/src/parse.rs b/orchid-host/src/parse.rs index 3e6a39b..d693c20 100644 --- a/orchid-host/src/parse.rs +++ b/orchid-host/src/parse.rs @@ -4,7 +4,7 @@ use futures::FutureExt; use futures::future::join_all; use itertools::Itertools; use never::Never; -use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err, mk_errv}; +use orchid_base::error::{OrcErrv, OrcRes, Reporter, ReporterImpl, mk_err, mk_errv}; use orchid_base::interner::Tok; use orchid_base::location::Pos; use orchid_base::macros::{MTok, MTree}; @@ -25,9 +25,19 @@ use crate::tree::{ type ParsSnippet<'a> = Snippet<'a, 'static, AtomHand, Never>; -pub trait ParseCtx: Send + Sync { +pub struct ParseCtxImpl<'a> { + pub systems: &'a [System], + pub reporter: &'a ReporterImpl, +} + +impl ParseCtx for ParseCtxImpl<'_> { + fn reporter(&self) -> &(impl Reporter + ?Sized) { self.reporter } + fn systems(&self) -> impl Iterator { self.systems.iter() } +} + +pub trait ParseCtx { fn systems(&self) -> impl Iterator; - fn reporter(&self) -> &impl Reporter; + fn reporter(&self) -> &(impl Reporter + ?Sized); } pub async fn parse_items( @@ -239,8 +249,8 @@ pub async fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes> { Token::LambdaHead(arg) => ( ttree.range.start..snip.pos().end, MTok::Lambda( - parse_mtree(Snippet::new(ttree, arg, snip.interner())).await?, - parse_mtree(tail).await?, + parse_mtree(Snippet::new(ttree, arg, snip.interner())).boxed_local().await?, + parse_mtree(tail).boxed_local().await?, ), Snippet::new(ttree, &[], snip.interner()), ), @@ -272,7 +282,7 @@ pub async fn parse_macro( let mut errors = Vec::new(); let mut rules = Vec::new(); for (i, item) in - line_items(Snippet::new(&prev, block, tail.interner())).await.into_iter().enumerate() + line_items(Snippet::new(prev, block, tail.interner())).await.into_iter().enumerate() { let Parsed { tail, output } = try_pop_no_fluff(item.tail).await?; if !output.is_kw(tail.i("rule").await) { diff --git a/orchid-host/src/rule/scal_match.rs b/orchid-host/src/rule/scal_match.rs index 98d0bfa..2c6a775 100644 --- a/orchid-host/src/rule/scal_match.rs +++ b/orchid-host/src/rule/scal_match.rs @@ -20,7 +20,7 @@ pub fn scal_match<'a>( (ScalMatcher::Placeh { key }, _) => Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))), (ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 => - any_match(&b_mat, &body[..], save_loc), + any_match(b_mat, &body[..], save_loc), (ScalMatcher::Lambda(arg_mat, b_mat), MacTok::Lambda(arg, body)) => Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)), _ => None, diff --git a/orchid-host/src/subprocess.rs b/orchid-host/src/subprocess.rs index 574cb40..6038251 100644 --- a/orchid-host/src/subprocess.rs +++ b/orchid-host/src/subprocess.rs @@ -1,7 +1,6 @@ use std::cell::RefCell; use std::path::PathBuf; use std::pin::Pin; -use std::rc::Rc; use std::thread; use async_process::{self, Child, ChildStdin, ChildStdout}; @@ -9,7 +8,6 @@ use async_std::io::{self, BufReadExt, BufReader}; use async_std::sync::Mutex; use futures::FutureExt; use futures::future::LocalBoxFuture; -use futures::task::LocalSpawnExt; use orchid_api_traits::{Decode, Encode}; use orchid_base::builtin::{ExtInit, ExtPort}; use orchid_base::logging::Logger; @@ -31,7 +29,7 @@ pub async fn ext_command( .stderr(async_process::Stdio::piped()) .spawn()?; let mut stdin = child.stdin.take().unwrap(); - api::HostHeader { log_strategy: logger.strat() }.encode(Pin::new(&mut stdin)); + api::HostHeader { log_strategy: logger.strat() }.encode(Pin::new(&mut stdin)).await; let mut stdout = child.stdout.take().unwrap(); let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await; let child_stderr = child.stderr.take().unwrap(); @@ -50,7 +48,7 @@ pub async fn ext_command( Ok(ExtInit { header, port: Box::new(Subprocess { - child: Rc::new(RefCell::new(child)), + child: RefCell::new(Some(child)), stdin: Mutex::new(Box::pin(stdin)), stdout: Mutex::new(Box::pin(stdout)), ctx, @@ -59,19 +57,18 @@ pub async fn ext_command( } pub struct Subprocess { - child: Rc>, + child: RefCell>, stdin: Mutex>>, stdout: Mutex>>, ctx: Ctx, } impl Drop for Subprocess { fn drop(&mut self) { - let child = self.child.clone(); - (self.ctx.spawn.spawn_local(async move { - let status = child.borrow_mut().status().await.expect("Extension exited with error"); + let mut child = self.child.borrow_mut().take().unwrap(); + (self.ctx.spawn)(Box::pin(async move { + let status = child.status().await.expect("Extension exited with error"); assert!(status.success(), "Extension exited with error {status}"); })) - .expect("Could not spawn process terminating future") } } impl ExtPort for Subprocess { diff --git a/orchid-host/src/system.rs b/orchid-host/src/system.rs index 09272a7..0bc0ba8 100644 --- a/orchid-host/src/system.rs +++ b/orchid-host/src/system.rs @@ -7,7 +7,6 @@ use async_stream::stream; use derive_destructure::destructure; use futures::StreamExt; use futures::future::join_all; -use futures::task::LocalSpawnExt; use hashbrown::HashMap; use itertools::Itertools; use orchid_base::async_once_cell::OnceCell; @@ -57,7 +56,7 @@ impl System { pub fn id(&self) -> api::SysId { self.0.id } pub fn ext(&self) -> &Extension { &self.0.ext } pub fn ctx(&self) -> &Ctx { &self.0.ctx } - pub(crate) fn reqnot(&self) -> &ReqNot { &self.0.ext.reqnot() } + pub(crate) fn reqnot(&self) -> &ReqNot { self.0.ext.reqnot() } pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind { self.reqnot().request(api::GetMember(self.0.id, id)).await } @@ -94,10 +93,9 @@ impl System { } pub(crate) fn drop_atom(&self, drop: api::AtomId) { let this = self.0.clone(); - (self.0.ctx.spawn.spawn_local(async move { + (self.0.ctx.spawn)(Box::pin(async move { this.ctx.owned_atoms.write().await.remove(&drop); })) - .expect("Failed to drop atom"); } pub async fn print(&self) -> String { let ctor = (self.0.ext.system_ctors().find(|c| c.id() == self.0.decl_id)) diff --git a/orchid-std/Cargo.toml b/orchid-std/Cargo.toml index c198131..9762ea7 100644 --- a/orchid-std/Cargo.toml +++ b/orchid-std/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" edition = "2021" [dependencies] +async-once-cell = "0.5.4" +async-std = "1.13.0" +futures = "0.3.31" itertools = "0.14.0" never = "0.1.0" once_cell = "1.20.2" @@ -13,3 +16,4 @@ orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" } orchid-base = { version = "0.1.0", path = "../orchid-base" } orchid-extension = { version = "0.1.0", path = "../orchid-extension" } ordered-float = "4.6.0" +tokio = { version = "1.43.0", features = ["full"] } diff --git a/orchid-std/src/main.rs b/orchid-std/src/main.rs index 6af59db..d4b9cf0 100644 --- a/orchid-std/src/main.rs +++ b/orchid-std/src/main.rs @@ -1,4 +1,16 @@ -use orchid_extension::entrypoint::ExtensionData; -use orchid_std::StdSystem; +use std::mem; +use std::rc::Rc; -pub fn main() { ExtensionData::new("orchid-std::main", &[&StdSystem]).main() } +use orchid_extension::entrypoint::{ExtensionData, extension_main_logic}; +use orchid_std::StdSystem; +use tokio::task::{LocalSet, spawn_local}; + +#[tokio::main(flavor = "current_thread")] +pub async fn main() { + LocalSet::new() + .run_until(async { + let data = ExtensionData::new("orchid-std::main", &[&StdSystem]); + extension_main_logic(data, Rc::new(|fut| mem::drop(spawn_local(fut)))).await; + }) + .await +} diff --git a/orchid-std/src/number/num_atom.rs b/orchid-std/src/number/num_atom.rs index 9c3458c..6ce9993 100644 --- a/orchid-std/src/number/num_atom.rs +++ b/orchid-std/src/number/num_atom.rs @@ -1,6 +1,8 @@ use orchid_api_derive::Coding; use orchid_base::error::OrcRes; -use orchid_extension::atom::{AtomFactory, Atomic, AtomicFeatures, MethodSetBuilder, ToAtom, TypAtom}; +use orchid_extension::atom::{ + AtomFactory, Atomic, AtomicFeatures, MethodSetBuilder, ToAtom, TypAtom, +}; use orchid_extension::atom_thin::{ThinAtom, ThinVariant}; use orchid_extension::conv::TryFromExpr; use orchid_extension::expr::Expr; @@ -15,8 +17,8 @@ impl Atomic for Int { } impl ThinAtom for Int {} impl TryFromExpr for Int { - fn try_from_expr(expr: Expr) -> OrcRes { - TypAtom::::try_from_expr(expr).map(|t| t.value) + async fn try_from_expr(expr: Expr) -> OrcRes { + TypAtom::::try_from_expr(expr).await.map(|t| t.value) } } @@ -29,8 +31,8 @@ impl Atomic for Float { } impl ThinAtom for Float {} impl TryFromExpr for Float { - fn try_from_expr(expr: Expr) -> OrcRes { - TypAtom::::try_from_expr(expr).map(|t| t.value) + async fn try_from_expr(expr: Expr) -> OrcRes { + TypAtom::::try_from_expr(expr).await.map(|t| t.value) } } @@ -39,10 +41,11 @@ pub enum Numeric { Float(NotNan), } impl TryFromExpr for Numeric { - fn try_from_expr(expr: Expr) -> OrcRes { - Int::try_from_expr(expr.clone()) - .map(|t| Numeric::Int(t.0)) - .or_else(|e| Float::try_from_expr(expr).map(|t| Numeric::Float(t.0)).map_err(|e2| e + e2)) + async fn try_from_expr(expr: Expr) -> OrcRes { + match Int::try_from_expr(expr.clone()).await { + Ok(t) => Ok(Numeric::Int(t.0)), + Err(e) => Float::try_from_expr(expr).await.map(|t| Numeric::Float(t.0)).map_err(|e2| e + e2), + } } } impl ToAtom for Numeric { diff --git a/orchid-std/src/number/num_lexer.rs b/orchid-std/src/number/num_lexer.rs index 010c6b1..a30277f 100644 --- a/orchid-std/src/number/num_lexer.rs +++ b/orchid-std/src/number/num_lexer.rs @@ -13,14 +13,14 @@ use super::num_atom::{Float, Int}; pub struct NumLexer; impl Lexer for NumLexer { const CHAR_FILTER: &'static [RangeInclusive] = &['0'..='9']; - fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> { + async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> { let ends_at = all.find(|c: char| !c.is_ascii_hexdigit() && !"xX._pP".contains(c)); let (chars, tail) = all.split_at(ends_at.unwrap_or(all.len())); let fac = match parse_num(chars) { Ok(Numeric::Float(f)) => Float(f).factory(), Ok(Numeric::Uint(uint)) => Int(uint.try_into().unwrap()).factory(), Ok(Numeric::Decimal(dec)) => Float(NotNan::new(dec.try_into().unwrap()).unwrap()).factory(), - Err(e) => return Err(num_to_err(e, ctx.pos(all)).into()), + Err(e) => return Err(num_to_err(e, ctx.pos(all), ctx.i).await.into()), }; Ok((tail, GenTok::X(fac).at(ctx.pos(all)..ctx.pos(tail)))) } diff --git a/orchid-std/src/std.rs b/orchid-std/src/std.rs index 088d7d2..1fe8317 100644 --- a/orchid-std/src/std.rs +++ b/orchid-std/src/std.rs @@ -1,7 +1,7 @@ -use std::sync::Arc; +use std::rc::Rc; use never::Never; -use orchid_base::interner::Tok; +use orchid_base::reqnot::Receipt; use orchid_extension::atom::{AtomDynfo, AtomicFeatures}; use orchid_extension::entrypoint::ExtReq; use orchid_extension::fs::DeclFs; @@ -31,15 +31,15 @@ impl SystemCard for StdSystem { } } impl System for StdSystem { - fn request(_: ExtReq, req: Self::Req) -> orchid_base::reqnot::Receipt { match req {} } + async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} } fn lexers() -> Vec { vec![&StringLexer] } fn parsers() -> Vec { vec![] } fn vfs() -> DeclFs { DeclFs::Mod(&[]) } - fn env() -> Vec<(Tok, MemKind)> { + fn env() -> Vec<(String, MemKind)> { vec![root_mod("std", [], [module(true, "string", [], [comments( ["Concatenate two strings"], - fun(true, "concat", |left: OrcString, right: OrcString| { - StrAtom::new(Arc::new(left.get_string().to_string() + &right.get_string())) + fun(true, "concat", |left: OrcString<'static>, right: OrcString<'static>| async move { + StrAtom::new(Rc::new(left.get_string().await.to_string() + &right.get_string().await)) }), )])])] } diff --git a/orchid-std/src/string/str_atom.rs b/orchid-std/src/string/str_atom.rs index f18b598..07b0ef2 100644 --- a/orchid-std/src/string/str_atom.rs +++ b/orchid-std/src/string/str_atom.rs @@ -1,43 +1,42 @@ use std::borrow::Cow; -use std::io; use std::ops::Deref; -use std::sync::Arc; +use std::pin::Pin; +use std::rc::Rc; +use async_std::io::Write; use orchid_api_derive::Coding; use orchid_api_traits::{Encode, Request}; use orchid_base::error::{OrcRes, mk_errv}; -use orchid_base::intern; -use orchid_base::interner::{Tok, intern}; +use orchid_base::interner::Tok; use orchid_extension::atom::{AtomMethod, Atomic, MethodSetBuilder, Supports, TypAtom}; use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant}; use orchid_extension::conv::TryFromExpr; use orchid_extension::expr::Expr; use orchid_extension::system::SysCtx; -#[derive(Copy, Clone, Coding)] +#[derive(Copy, Clone, Debug, Coding)] pub struct StringGetVal; impl Request for StringGetVal { - type Response = Arc; + type Response = Rc; } impl AtomMethod for StringGetVal { const NAME: &str = "std::string_get_val"; } impl Supports for StrAtom { - fn handle(&self, _: SysCtx, _: StringGetVal) -> ::Response { + async fn handle(&self, _: SysCtx, _: StringGetVal) -> ::Response { self.0.clone() } } #[derive(Clone)] -pub struct StrAtom(Arc); +pub struct StrAtom(Rc); impl Atomic for StrAtom { type Variant = OwnedVariant; type Data = (); fn reg_reqs() -> MethodSetBuilder { MethodSetBuilder::new().handle::() } } impl StrAtom { - pub fn new(str: Arc) -> Self { Self(str) } - pub fn value(&self) -> Arc { self.0.clone() } + pub fn new(str: Rc) -> Self { Self(str) } } impl Deref for StrAtom { type Target = str; @@ -45,12 +44,12 @@ impl Deref for StrAtom { } impl OwnedAtom for StrAtom { type Refs = (); - fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) } - fn serialize(&self, _: SysCtx, sink: &mut (impl io::Write + ?Sized)) -> Self::Refs { - self.deref().encode(sink) + async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) } + async fn serialize(&self, _: SysCtx, sink: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs { + self.deref().encode(sink).await } - fn deserialize(mut ctx: impl DeserializeCtx, _: Self::Refs) -> Self { - Self::new(Arc::new(ctx.read::())) + async fn deserialize(mut ctx: impl DeserializeCtx, _: Self::Refs) -> Self { + Self::new(Rc::new(ctx.read::().await)) } } @@ -66,34 +65,46 @@ impl From> for IntStrAtom { } impl OwnedAtom for IntStrAtom { type Refs = (); - fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.to_api()) } - fn print(&self, _ctx: SysCtx) -> String { format!("{:?}i", *self.0) } - fn serialize(&self, _: SysCtx, write: &mut (impl io::Write + ?Sized)) { self.0.encode(write) } - fn deserialize(ctx: impl DeserializeCtx, _: ()) -> Self { Self(intern(&ctx.decode::())) } + async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.to_api()) } + async fn print(&self, _ctx: SysCtx) -> String { format!("{:?}i", *self.0) } + async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) { + self.0.encode(write).await + } + async fn deserialize(mut ctx: impl DeserializeCtx, _: ()) -> Self { + let s = ctx.decode::().await; + Self(ctx.sys().i.i(&s).await) + } } #[derive(Clone)] -pub enum OrcString<'a> { +pub struct OrcString<'a> { + kind: OrcStringKind<'a>, + ctx: SysCtx, +} + +#[derive(Clone)] +pub enum OrcStringKind<'a> { Val(TypAtom<'a, StrAtom>), Int(TypAtom<'a, IntStrAtom>), } impl OrcString<'_> { - pub fn get_string(&self) -> Arc { - match &self { - Self::Int(tok) => Tok::from_api(tok.value).arc(), - Self::Val(atom) => atom.request(StringGetVal), + pub async fn get_string(&self) -> Rc { + match &self.kind { + OrcStringKind::Int(tok) => self.ctx.i.ex(**tok).await.rc(), + OrcStringKind::Val(atom) => atom.request(StringGetVal).await, } } } impl TryFromExpr for OrcString<'static> { - fn try_from_expr(expr: Expr) -> OrcRes> { - if let Ok(v) = TypAtom::::try_from_expr(expr.clone()) { - return Ok(OrcString::Val(v)); + async fn try_from_expr(expr: Expr) -> OrcRes> { + if let Ok(v) = TypAtom::::try_from_expr(expr.clone()).await { + return Ok(OrcString { ctx: expr.ctx(), kind: OrcStringKind::Val(v) }); } - match TypAtom::::try_from_expr(expr) { - Ok(t) => Ok(OrcString::Int(t)), - Err(e) => Err(mk_errv(intern!(str: "A string was expected"), "", e.pos_iter())), + let ctx = expr.ctx(); + match TypAtom::::try_from_expr(expr).await { + Ok(t) => Ok(OrcString { ctx: t.data.ctx(), kind: OrcStringKind::Int(t) }), + Err(e) => Err(mk_errv(ctx.i.i("A string was expected").await, "", e.pos_iter())), } } } diff --git a/orchid-std/src/string/str_lexer.rs b/orchid-std/src/string/str_lexer.rs index ff9434e..503a5fe 100644 --- a/orchid-std/src/string/str_lexer.rs +++ b/orchid-std/src/string/str_lexer.rs @@ -1,9 +1,9 @@ use itertools::Itertools; use orchid_base::error::{OrcErr, OrcRes, mk_err, mk_errv}; -use orchid_base::interner::intern; +use orchid_base::interner::Interner; use orchid_base::location::Pos; use orchid_base::tree::{vname_tv, wrap_tokv}; -use orchid_base::{intern, vname}; +use orchid_base::vname; use orchid_extension::atom::AtomicFeatures; use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable}; use orchid_extension::tree::{GenTok, GenTokTree}; @@ -32,10 +32,10 @@ struct StringError { impl StringError { /// Convert into project error for reporting - pub fn into_proj(self, pos: u32) -> OrcErr { + pub async fn into_proj(self, pos: u32, i: &Interner) -> OrcErr { let start = pos + self.pos; mk_err( - intern!(str: "Failed to parse string"), + i.i("Failed to parse string").await, match self.kind { StringErrorKind::NotHex => "Expected a hex digit", StringErrorKind::BadCodePoint => "The specified number is not a Unicode code point", @@ -95,29 +95,40 @@ fn parse_string(str: &str) -> Result { pub struct StringLexer; impl Lexer for StringLexer { const CHAR_FILTER: &'static [std::ops::RangeInclusive] = &['"'..='"']; - fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> { - let mut tail = all.strip_prefix('"').ok_or_else(err_not_applicable)?; - let mut ret = GenTok::X(IntStrAtom::from(intern!(str: "")).factory()).at(ctx.tok_ran(0, all)); + async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> { + let Some(mut tail) = all.strip_prefix('"') else { + return Err(err_not_applicable(ctx.i).await.into()); + }; + let mut ret = GenTok::X(IntStrAtom::from(ctx.i.i("").await).factory()).at(ctx.tok_ran(0, all)); let mut cur = String::new(); let mut errors = vec![]; - let str_to_gen = |str: &mut String, tail: &str, err: &mut Vec| { - let str_val = parse_string(&str.split_off(0)) - .inspect_err(|e| err.push(e.clone().into_proj(ctx.pos(tail) - str.len() as u32))) - .unwrap_or_default(); - GenTok::X(IntStrAtom::from(intern(&*str_val)).factory()) - .at(ctx.tok_ran(str.len() as u32, tail)) - }; - let add_frag = |prev: GenTokTree<'a>, new: GenTokTree<'a>| { - wrap_tokv(vname_tv(&vname!(std::string::concat), new.range.end).chain([prev, new])) + async fn str_to_gen<'a>( + str: &mut String, + tail: &str, + err: &mut Vec, + ctx: &'a LexContext<'a>, + ) -> GenTokTree<'a> { + let str_val_res = parse_string(&str.split_off(0)); + if let Err(e) = &str_val_res { + err.push(e.clone().into_proj(ctx.pos(tail) - str.len() as u32, ctx.i).await); + } + let str_val = str_val_res.unwrap_or_default(); + GenTok::X(IntStrAtom::from(ctx.i.i(&*str_val).await).factory()) + .at(ctx.tok_ran(str.len() as u32, tail)) as GenTokTree<'a> + } + let add_frag = |prev: GenTokTree<'a>, new: GenTokTree<'a>| async { + wrap_tokv( + vname_tv(&vname!(std::string::concat; ctx.i).await, new.range.end).chain([prev, new]), + ) }; loop { if let Some(rest) = tail.strip_prefix('"') { - return Ok((rest, add_frag(ret, str_to_gen(&mut cur, tail, &mut errors)))); + return Ok((rest, add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await)); } else if let Some(rest) = tail.strip_prefix('$') { - ret = add_frag(ret, str_to_gen(&mut cur, tail, &mut errors)); - let (new_tail, tree) = ctx.recurse(rest)?; + ret = add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await; + let (new_tail, tree) = ctx.recurse(rest).await?; tail = new_tail; - ret = add_frag(ret, tree); + ret = add_frag(ret, tree).await; } else if tail.starts_with('\\') { // parse_string will deal with it, we just have to skip the next char tail = &tail[2..]; @@ -128,9 +139,11 @@ impl Lexer for StringLexer { tail = ch.as_str(); } else { let range = ctx.pos(all)..ctx.pos(""); - return Err(mk_errv(intern!(str: "No string end"), "String never terminated with \"", [ - Pos::Range(range.clone()).into(), - ])); + return Err(mk_errv( + ctx.i.i("No string end").await, + "String never terminated with \"", + [Pos::Range(range.clone()).into()], + )); } } } diff --git a/orcx/Cargo.toml b/orcx/Cargo.toml index 9757b83..f1005b3 100644 --- a/orcx/Cargo.toml +++ b/orcx/Cargo.toml @@ -6,8 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-stream = "0.3.6" camino = "1.1.9" clap = { version = "4.5.24", features = ["derive"] } +futures = "0.3.31" itertools = "0.14.0" orchid-base = { version = "0.1.0", path = "../orchid-base" } orchid-host = { version = "0.1.0", path = "../orchid-host" } +substack = "1.1.1" +tokio = { version = "1.43.0", features = ["full"] } diff --git a/orcx/src/main.rs b/orcx/src/main.rs index c67964d..e85697a 100644 --- a/orcx/src/main.rs +++ b/orcx/src/main.rs @@ -1,17 +1,26 @@ use std::fs::File; use std::io::Read; +use std::mem; use std::process::Command; -use std::sync::Arc; +use std::rc::Rc; +use async_stream::try_stream; use camino::Utf8PathBuf; use clap::{Parser, Subcommand}; -use itertools::Itertools; -use orchid_base::interner::intern; +use futures::{Stream, TryStreamExt, io}; +use orchid_base::clone; +use orchid_base::error::ReporterImpl; use orchid_base::logging::{LogStrategy, Logger}; +use orchid_base::parse::Snippet; use orchid_base::tree::ttv_fmt; -use orchid_host::extension::{Extension, init_systems}; +use orchid_host::ctx::Ctx; +use orchid_host::extension::Extension; use orchid_host::lex::lex; -use orchid_host::subprocess::Subprocess; +use orchid_host::parse::{self, ParseCtx, ParseCtxImpl, parse_items}; +use orchid_host::subprocess::ext_command; +use orchid_host::system::init_systems; +use substack::Substack; +use tokio::task::{LocalSet, spawn_local}; #[derive(Parser, Debug)] #[command(version, about, long_about)] @@ -30,23 +39,65 @@ pub enum Commands { #[arg(short, long)] file: Utf8PathBuf, }, + Parse { + #[arg(short, long)] + file: Utf8PathBuf, + }, } -fn main() { - let args = Args::parse(); - let logger = Logger::new(LogStrategy::StdErr); - match args.command { - Commands::Lex { file } => { - let extensions = (args.extension.iter()) - .map(|f| Subprocess::new(Command::new(f.as_os_str()), logger.clone()).unwrap()) - .map(|cmd| Extension::new(Arc::new(cmd), logger.clone()).unwrap()) - .collect_vec(); - let systems = init_systems(&args.system, &extensions).unwrap(); - let mut file = File::open(file.as_std_path()).unwrap(); - let mut buf = String::new(); - file.read_to_string(&mut buf).unwrap(); - let lexemes = lex(intern(&buf), &systems).unwrap(); - println!("{}", ttv_fmt(&lexemes)) - }, +fn get_all_extensions<'a>( + args: &'a Args, + logger: &'a Logger, + ctx: &'a Ctx, +) -> impl Stream> + 'a { + try_stream! { + for exe in args.extension.iter() { + let init = ext_command(Command::new(exe.as_os_str()), logger.clone(), ctx.clone()).await + .unwrap(); + let ext = Extension::new(init, logger.clone(), ctx.clone())?; + spawn_local(clone!(ext; async move { loop { ext.recv_one().await }})); + yield ext + } } } + +#[tokio::main(flavor = "current_thread")] +async fn main() { + LocalSet::new() + .run_until(async { + let args = Args::parse(); + let ctx = &Ctx::new(Rc::new(|fut| mem::drop(spawn_local(fut)))); + let logger = Logger::new(LogStrategy::Discard); + let extensions = + get_all_extensions(&args, &logger, ctx).try_collect::>().await.unwrap(); + match args.command { + Commands::Lex { file } => { + let systems = init_systems(&args.system, &extensions).await.unwrap(); + let mut file = File::open(file.as_std_path()).unwrap(); + let mut buf = String::new(); + file.read_to_string(&mut buf).unwrap(); + let lexemes = lex(ctx.i.i(&buf).await, &systems, ctx).await.unwrap(); + println!("{}", ttv_fmt(&lexemes).await) + }, + Commands::Parse { file } => { + let systems = init_systems(&args.system, &extensions).await.unwrap(); + let mut file = File::open(file.as_std_path()).unwrap(); + let mut buf = String::new(); + file.read_to_string(&mut buf).unwrap(); + let lexemes = lex(ctx.i.i(&buf).await, &systems, ctx).await.unwrap(); + let Some(first) = lexemes.first() else { + println!("File empty!"); + return; + }; + let reporter = ReporterImpl::new(); + let pctx = ParseCtxImpl { reporter: &reporter, systems: &systems }; + let snip = Snippet::new(first, &lexemes, &ctx.i); + let ptree = parse_items(&pctx, Substack::Bottom, snip).await.unwrap(); + for item in ptree { + println!("{item:?}") + } + }, + } + }) + .await +} diff --git a/test-log.txt b/test-log.txt new file mode 100644 index 0000000..d3a6756 --- /dev/null +++ b/test-log.txt @@ -0,0 +1 @@ +Upsending: [ff ff ff ff ff ff ff f7 00 00 00 00 00 00 00 08 22 75 73 65 72 21 22 69]