Orchid/orchid
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Orchid:9e7648bc72ab2d0474229777a9a1a05c07fda6c6
Branches Tags
Orchid:main Orchid:ipc-refactor Orchid:ctx-refactor
...
compare: Orchid:19f2c6426afc71149b3d1ec25c087e7357a5121c
Branches Tags
Orchid:main Orchid:ipc-refactor Orchid:ctx-refactor

10 Commits
9e7648bc72 ... 19f2c6426a

Author SHA1 Message Date
Lawrence Bethlenfalvy
19f2c6426a Returned from Italy 2025-07-18 16:29:52 +02:00
Lawrence Bethlenfalvy
fe89188c4b temp commit 2025-07-12 00:46:10 +02:00
Lawrence Bethlenfalvy
1868f1a506 moving back 2025-05-04 16:37:32 +01:00
Lawrence Bethlenfalvy
1543aa8c13 commit before travel 2025-05-01 22:21:37 +02:00
Lawrence Bethlenfalvy
1a25f52471 Commit pending merge 2025-04-30 22:18:09 +02:00
Lawrence Bethlenfalvy
c9b349bccf sync commit 2025-04-23 16:01:22 +01:00
Lawrence Bethlenfalvy
94958bfbf5 commit before easter break 2025-04-15 00:34:45 +02:00
Lawrence Bethlenfalvy
f783445a76 fixed link in readme 2025-02-23 01:36:09 +01:00
Lawrence Bethlenfalvy
5e474069e0 reference cycles resolved 2025-02-22 19:01:05 +01:00
Lawrence Bethlenfalvy
cfa8b6ee52 for a moment, everything works 2025-02-21 23:53:31 +01:00
107 changed files with 3874 additions and 3293 deletions
Expand all files Collapse all files

2
.cargo/config.toml
View File

@@ -1,9 +1,11 @@
[alias]
xtask = "run --quiet --package xtask --"
orcx = "xtask orcx"
orcxdb = "xtask orcxdb"
[env]
CARGO_WORKSPACE_DIR = { value = "", relative = true }
ORCHID_EXTENSIONS = "target/debug/orchid-std"
ORCHID_DEFAULT_SYSTEMS = "orchid::std"
ORCHID_LOG_BUFFERS = "true"
RUSTBACKTRACE = "1"

77
Cargo.lock generated
View File

@@ -862,6 +862,25 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "include_dir"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd"
dependencies = [
"include_dir_macros",
]
[[package]]
name = "include_dir_macros"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75"
dependencies = [
"proc-macro2 1.0.92",
"quote 1.0.38",
]
[[package]]
name = "indexmap"
version = "2.7.0"
@@ -872,6 +891,17 @@ dependencies = [
"hashbrown 0.15.2",
]
[[package]]
name = "io-uring"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
dependencies = [
"bitflags",
"cfg-if",
"libc",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
@@ -1124,7 +1154,6 @@ dependencies = [
"ordered-float",
"regex",
"rust-embed",
"rust_decimal",
"some_executor 0.4.0",
"substack",
"test_executors",
@@ -1138,10 +1167,12 @@ dependencies = [
"ahash 0.8.11",
"async-once-cell",
"async-std",
"async-stream",
"derive_destructure",
"dyn-clone",
"futures",
"hashbrown 0.15.2",
"include_dir",
"itertools",
"konst",
"lazy_static",
@@ -1153,9 +1184,10 @@ dependencies = [
"orchid-api-traits",
"orchid-base",
"ordered-float",
"paste",
"some_executor 0.4.0",
"pastey",
"some_executor 0.5.1",
"substack",
"tokio",
"trait-set",
]
@@ -1173,6 +1205,7 @@ dependencies = [
"hashbrown 0.15.2",
"itertools",
"lazy_static",
"memo-map",
"never",
"num-traits",
"orchid-api",
@@ -1185,13 +1218,19 @@ dependencies = [
"trait-set",
]
[[package]]
name = "orchid-macros"
version = "0.1.0"
[[package]]
name = "orchid-std"
version = "0.1.0"
dependencies = [
"async-once-cell",
"async-std",
"async-stream",
"futures",
"hashbrown 0.15.2",
"itertools",
"never",
"once_cell",
@@ -1201,6 +1240,8 @@ dependencies = [
"orchid-base",
"orchid-extension",
"ordered-float",
"rust_decimal",
"test_executors",
"tokio",
]
@@ -1223,9 +1264,9 @@ dependencies = [
[[package]]
name = "ordered-float"
version = "4.6.0"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
checksum = "e2c1f9f56e534ac6a9b8a4600bdf0f530fb393b5f393e7b4d03489c3cf0c3f01"
dependencies = [
"num-traits",
]
@@ -1265,6 +1306,12 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pastey"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a8cb46bdc156b1c90460339ae6bfd45ba0394e5effbaa640badb4987fdc261"
[[package]]
name = "pin-project-lite"
version = "0.2.16"
@@ -1699,6 +1746,20 @@ dependencies = [
"web-time",
]
[[package]]
name = "some_executor"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb323f85458f395c28b5ea35a2626e9f46a35f1d730d37e6fa24dcf2848835ee"
dependencies = [
"atomic-waker",
"priority",
"wasm-bindgen",
"wasm_thread",
"web-sys",
"web-time",
]
[[package]]
name = "stdio-perftest"
version = "0.1.0"
@@ -1796,17 +1857,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.43.0"
version = "1.46.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
dependencies = [
"backtrace",
"bytes",
"io-uring",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"slab",
"socket2",
"tokio-macros",
"windows-sys 0.52.0",

2
Cargo.toml
View File

@@ -10,5 +10,5 @@ members = [
"orchid-api",
"orchid-api-derive",
"orchid-api-traits",
"stdio-perftest", "xtask",
"stdio-perftest", "xtask", "orchid-macros",
]

4
README.md
View File

@@ -7,7 +7,7 @@ An experimental lazy, pure functional programming language designed to be embedd
## Usage
The standalone interpreter can be built as the binary target from this package. The language tutorial and standard library documentation is at [www.lbfalvy.com/orchid-reference](https://www.lbfalvy.com/orchid-reference/). Embedder guide and Rust API documentation are coming soon.
The standalone interpreter can be built as the binary target from this package. The language tutorial and standard library documentation is at [www.lbfalvy.com/orchid-reference](https://lbfalvy.github.io/orchid-reference/). Embedder guide and Rust API documentation are coming soon.
## Design
@@ -43,4 +43,4 @@ Identifying marks include the Orchid logo, the ribbon image above, and the names
Contact information includes email addresses, links to the source code and issue tracker.
Words listed as identifying marks are explicltly not considered as such when they apear in technical interfaces or APIs. For example, shell commands, identifiers within the language,
Words listed as identifying marks are explicltly not considered as such when they appear in technical interfaces or APIs. For example, shell commands, identifiers within Orchid or Rust code, and names in package registries are not considered as identifying marks.

16
SWAP.md
View File

@@ -1,16 +1,10 @@
## Async conversion
Since the macro AST is built as a custom tokenizer inside the system, it needs access to the import set. On the other hand, import sets aren't available until after parsing. Need a way to place this order in a lexer without restricting the expression value of the lexer.
convert host to async non-send
demonstrate operation with existing lex-hello example
consider converting extension's SysCtx to a typed context bag
align fn atom and macros on both sides with new design. No global state.
The daft option of accepting import resolution queries at runtime is available but consider better options.
## alternate extension mechanism
The Macro extension needs to be in the same compilation unit as the interpreter because the interpreter needs to proactively access its datastructures (in particular, it needs to generate MacTree from TokTree)
The STD system will have a lot of traffic for trivial operations like algebra, stream IO will likely not be fast enough. A faster system is in order.
Ideally, it should reuse `orchid-extension` for message routing and decoding.
@@ -18,10 +12,8 @@ Ideally, it should reuse `orchid-extension` for message routing and decoding.
## Preprocessor extension
Must figure out how preprocessor can both be a System and referenced in the interpreter
The macro system will not be privileged, it can take control from the interpreter via a custom top-level "let" line type.
Must actually write macro system as recorded in note
At this point swappable preprocessors aren't a target because interaction with module system sounds complicated
Check if any of this needs interpreter, if so, start with that

7
notes/new_macro_model.md
View File

@@ -42,9 +42,7 @@ Prioritised macro patterns must start and end with a vectorial placeholder. They
Macros are checked from the outermost block inwards.
1. For every name token, test all named macros starting with that name
1. Take the first rule that matches in each block
2. If there are multiple matches across blocks, raise an ambiguity error
3. If the tail is implicit, recurse on it
1. If the tail is implicit, continue iterating
2. Test all prioritized macros
1. Take the first rule that matches in the highest prioritized block
@@ -78,7 +76,8 @@ Recursion has to happen through the interpreter itself, so the macro system is d
- atom `MacRecurState` holds the recursion state
- function `resolve_recur` finds all matches on a MacTree
- type: `MacRecurState -> MacTree -> MacTree`
- use all macros to find all matches in the tree
- use all relevant macros to find all matches in the tree
- since macros must contain a locally defined token, it can be assumed that at the point that a constant is evaluated and all imports in the parent module have been resolved, necessarily all relevant macro rules must have been loaded
- for each match
- check for recursion violations
- wrap the body in iife-s corresponding to the named values in the match state

2
orchid-api-derive/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "orchid-api-derive"
version = "0.1.0"
edition = "2021"
edition = "2024"
[lib]
proc-macro = true

4
orchid-api-traits/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "orchid-api-traits"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -11,4 +11,4 @@ async-stream = "0.3.6"
futures = "0.3.31"
itertools = "0.14.0"
never = "0.1.0"
ordered-float = "4.6.0"
ordered-float = "5.0.0"

10
orchid-api-traits/src/coding.rs
View File

@@ -9,8 +9,7 @@ use std::sync::Arc;
use async_std::io::{Read, ReadExt, Write, WriteExt};
use async_stream::stream;
use futures::future::LocalBoxFuture;
use futures::{FutureExt, StreamExt};
use futures::StreamExt;
use never::Never;
use ordered_float::NotNan;
@@ -28,11 +27,8 @@ pub trait Encode {
pub trait Coding: Encode + Decode + Clone {
fn get_decoder<T: 'static, F: Future<Output = T> + 'static>(
map: impl Fn(Self) -> F + Clone + 'static,
) -> impl for<'a> Fn(Pin<&'a mut dyn Read>) -> LocalBoxFuture<'a, T> {
move |r| {
let map = map.clone();
async move { map(Self::decode(r).await).await }.boxed_local()
}
) -> impl AsyncFn(Pin<&mut dyn Read>) -> T {
async move |r| map(Self::decode(r).await).await
}
}
impl<T: Encode + Decode + Clone> Coding for T {}

4
orchid-api/Cargo.toml
View File

@@ -1,12 +1,12 @@
[package]
name = "orchid-api"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
ordered-float = "4.6.0"
ordered-float = "5.0.0"
orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
async-std = "1.13.0"

6
orchid-api/src/lexer.rs
View File

@@ -3,7 +3,7 @@ use std::ops::RangeInclusive;
use orchid_api_derive::{Coding, Hierarchy};
use orchid_api_traits::Request;
use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TokenTree, TreeTicket};
use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TStrv, TokenTree};
/// - All ranges contain at least one character
/// - All ranges are in increasing characeter order
@@ -18,6 +18,8 @@ pub struct LexExpr {
pub id: ParsId,
pub text: TStr,
pub pos: u32,
/// Source root module path
pub src: TStrv,
}
impl Request for LexExpr {
type Response = Option<OrcResult<LexedExpr>>;
@@ -42,5 +44,5 @@ impl Request for SubLex {
#[derive(Clone, Debug, Coding)]
pub struct SubLexed {
pub pos: u32,
pub ticket: TreeTicket,
pub tree: TokenTree,
}

4
orchid-api/src/lib.rs
View File

@@ -2,8 +2,6 @@ mod lexer;
pub use lexer::*;
mod format;
pub use format::*;
mod macros;
pub use macros::*;
mod atom;
pub use atom::*;
mod error;
@@ -24,5 +22,3 @@ mod system;
pub use system::*;
mod tree;
pub use tree::*;
mod vfs;
pub use vfs::*;

2
orchid-api/src/location.rs
View File

@@ -17,8 +17,6 @@ pub enum Location {
Gen(CodeGenInfo),
/// Range and file
SourceRange(SourceRange),
/// Range only, file implied. Most notably used by parsers
Range(Range<u32>),
}
#[derive(Clone, Debug, Coding)]

86
orchid-api/src/macros.rs
View File

@@ -1,86 +0,0 @@
use std::collections::HashMap;
use std::num::NonZeroU64;
use orchid_api_derive::{Coding, Hierarchy};
use orchid_api_traits::Request;
use ordered_float::NotNan;
use crate::{
Atom, Comment, ExtHostReq, HostExtReq, Location, OrcResult, Paren, ParsId, SysId, TStr, TStrv,
};
#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MacroTreeId(pub NonZeroU64);
#[derive(Clone, Debug, Coding)]
pub struct MacroTree {
pub location: Location,
pub token: MacroToken,
}
#[derive(Clone, Debug, Coding)]
pub enum MacroToken {
S(Paren, Vec<MacroTree>),
Name(TStrv),
Slot(MacroTreeId),
Lambda(Vec<MacroTree>, Vec<MacroTree>),
Ph(Placeholder),
Atom(Atom),
}
#[derive(Clone, Debug, Coding)]
pub struct MacroBlock {
pub priority: Option<NotNan<f64>>,
pub rules: Vec<MacroRule>,
}
#[derive(Clone, Debug, Coding)]
pub struct MacroRule {
pub location: Location,
pub comments: Vec<Comment>,
pub pattern: Vec<MacroTree>,
pub id: MacroId,
}
/// A specific macro rule with a specific pattern across invocations
#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, Hash)]
pub struct MacroId(pub NonZeroU64);
/// After a pattern matches, this call executes the body of the macro. This
/// request returns None if an inner nested request raised an exception
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(HostExtReq)]
pub struct ApplyMacro {
pub sys: SysId,
pub id: MacroId,
/// Recursion token
pub run_id: ParsId,
/// Must contain exactly the keys that were specified as placeholders in the
/// pattern
pub params: HashMap<TStr, Vec<MacroTree>>,
}
impl Request for ApplyMacro {
type Response = Option<OrcResult<Vec<MacroTree>>>;
}
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(ExtHostReq)]
pub struct RunMacros {
pub run_id: ParsId,
pub query: Vec<MacroTree>,
}
impl Request for RunMacros {
type Response = Option<Vec<MacroTree>>;
}
#[derive(Clone, Debug, Coding)]
pub struct Placeholder {
pub name: TStr,
pub kind: PhKind,
}
#[derive(Clone, Copy, Debug, Coding)]
pub enum PhKind {
Scalar,
Vector { priority: u8, at_least_one: bool },
}

96
orchid-api/src/parser.rs
View File

@@ -1,28 +1,104 @@
use core::ops::Range;
use std::num::NonZeroU64;
use orchid_api_derive::{Coding, Decode, Encode, Hierarchy};
use orchid_api_derive::{Coding, Hierarchy};
use orchid_api_traits::Request;
use crate::{Comment, HostExtReq, OrcResult, SysId, TokenTree};
use crate::{
Expression, ExtHostReq, HostExtReq, OrcResult, SourceRange, SysId, TStr, TStrv, TokenTree,
};
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
pub struct ParsId(pub NonZeroU64);
// impl orchid_api_traits::Decode for ParsId {
// async fn decode<R: async_std::io::Read + ?Sized>(mut read:
// std::pin::Pin<&mut R>) -> Self {
// Self(orchid_api_traits::Decode::decode(read.as_mut()).await)
// }
// }
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
pub struct ParsedConstId(pub NonZeroU64);
/// Parse a single source line. Return values can be modules, constants, or
/// token sequences for re-parsing. These re-parsed token sequences can also
/// represent raw language items such as modules, imports, and const. This is
/// how we enable generating imports without forcing import syntax to affect API
/// versioning
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(HostExtReq)]
pub struct ParseLine {
pub sys: SysId,
/// The immediately enclosing module path
pub module: TStrv,
/// The root module path for the snipppet of source code, prefix of
/// [ParseLine#module]
pub src: TStrv,
pub comments: Vec<Comment>,
pub exported: bool,
pub line: Vec<TokenTree>,
}
impl Request for ParseLine {
type Response = OrcResult<Vec<TokenTree>>;
type Response = OrcResult<Vec<ParsedLine>>;
}
#[derive(Clone, Debug, Coding)]
pub struct ParsedLine {
pub comments: Vec<Comment>,
pub source_range: SourceRange,
pub kind: ParsedLineKind,
}
#[derive(Clone, Debug, Coding)]
pub enum ParsedLineKind {
Recursive(Vec<TokenTree>),
Member(ParsedMember),
}
#[derive(Clone, Debug, Coding)]
pub struct ParsedMember {
pub name: TStr,
pub exported: bool,
pub kind: ParsedMemberKind,
}
#[derive(Clone, Debug, Coding)]
pub enum ParsedMemberKind {
Constant(ParsedConstId),
Module(Vec<ParsedLine>),
}
/// Obtain the value of a parsed constant. This is guaranteed to be called after
/// the last [ParseLine] but before any [crate::AtomReq]. As such, in principle
/// the macro engine could run here.
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(HostExtReq)]
pub struct FetchParsedConst {
pub sys: SysId,
pub id: ParsedConstId,
}
impl Request for FetchParsedConst {
type Response = Expression;
}
#[derive(Clone, Debug, Coding)]
pub struct Comment {
pub text: TStr,
pub range: Range<u32>,
}
/// Resolve relative names from the perspective of a constant. This can only be
/// called during a [FetchParsedConst] call, but it can be called for a
/// different [ParsedConstId] from the one in [FetchParsedConst].
///
/// Each name is either resolved to an alias or existing constant `Some(TStrv)`
/// or not resolved `None`. An error is never raised, as names may have a
/// primary meaning such as a local binding which can be overridden by specific
/// true names such as those triggering macro keywords. It is not recommended to
/// define syntax that can break by defining arbitrary constants, as line
/// parsers can define new ones at will.
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(ExtHostReq)]
pub struct ResolveNames {
pub sys: SysId,
pub constid: ParsedConstId,
pub names: Vec<TStrv>,
}
impl Request for ResolveNames {
type Response = Vec<Option<TStrv>>;
}

8
orchid-api/src/proto.rs
View File

@@ -28,7 +28,7 @@ use async_std::io::{Read, Write};
use orchid_api_derive::{Coding, Hierarchy};
use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
use crate::{atom, expr, interner, lexer, logging, macros, parser, system, tree, vfs};
use crate::{atom, expr, interner, lexer, logging, parser, system, tree};
static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
pub struct HostHeader {
@@ -88,7 +88,8 @@ pub enum ExtHostReq {
SysFwd(system::SysFwd),
ExprReq(expr::ExprReq),
SubLex(lexer::SubLex),
RunMacros(macros::RunMacros),
LsModule(tree::LsModule),
ResolveNames(parser::ResolveNames),
}
/// Notifications sent from the extension to the host
@@ -117,9 +118,8 @@ pub enum HostExtReq {
DeserAtom(atom::DeserAtom),
LexExpr(lexer::LexExpr),
ParseLine(parser::ParseLine),
FetchParsedConst(parser::FetchParsedConst),
GetMember(tree::GetMember),
VfsReq(vfs::VfsReq),
ApplyMacro(macros::ApplyMacro),
}
/// Notifications sent from the host to the extension

4
orchid-api/src/system.rs
View File

@@ -52,11 +52,11 @@ pub struct NewSystem {
pub depends: Vec<SysId>,
}
impl Request for NewSystem {
type Response = SystemInst;
type Response = NewSystemResponse;
}
#[derive(Clone, Debug, Coding)]
pub struct SystemInst {
pub struct NewSystemResponse {
/// The set of possible starting characters of tokens the lexer of this system
/// can process. The lexer will notify this system if it encounters one of
/// these characters.9

98
orchid-api/src/tree.rs
View File

@@ -1,14 +1,12 @@
use std::collections::HashMap;
use std::num::NonZeroU64;
use std::ops::Range;
use std::sync::Arc;
use std::rc::Rc;
use orchid_api_derive::{Coding, Hierarchy};
use orchid_api_traits::Request;
use ordered_float::NotNan;
use crate::{
Atom, Expression, HostExtReq, Location, MacroBlock, OrcError, Placeholder, SysId, TStr, TStrv,
};
use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TStr, TStrv};
/// A token tree from a lexer recursion request. Its lifetime is the lex call,
/// the lexer can include it in its output or discard it by implication.
@@ -32,27 +30,24 @@ pub enum Token {
LambdaHead(Vec<TokenTree>),
/// A name segment or an operator.
Name(TStr),
/// An absolute name
Reference(TStrv),
/// A newly generated expression. The last place this is supposed to happen is
/// in lexers, parsers and macros should have enumerable many outputs
/// expressed as function calls.
NewExpr(Expression),
/// A pre-existing expression
Handle(ExprTicket),
/// ::
NS,
NS(TStr, Box<TokenTree>),
/// Line break.
BR,
/// ( Round parens ), [ Square brackets ] or { Curly braces }
S(Paren, Vec<TokenTree>),
/// A new atom
Atom(Atom),
/// Anchor to insert a subtree
Slot(TreeTicket),
/// A static compile-time error returned by failing lexers if
/// the rest of the source is likely still meaningful
/// the rest of the source is likely still meaningful. This is distinct from
/// NewExpr(Bottom) because it fails in dead branches too.
Bottom(Vec<OrcError>),
/// A comment
Comment(Arc<String>),
/// Placeholder
Ph(Placeholder),
/// Macro block head
Macro(Option<NotNan<f64>>),
Comment(Rc<String>),
}
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]
@@ -65,31 +60,12 @@ pub enum Paren {
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
pub struct TreeId(pub NonZeroU64);
#[derive(Clone, Debug, Coding)]
pub struct Item {
pub location: Location,
pub comments: Vec<Comment>,
pub kind: ItemKind,
}
#[derive(Clone, Debug, Coding)]
pub enum ItemKind {
Member(Member),
Macro(MacroBlock),
Export(TStr),
Import(TStrv),
}
#[derive(Clone, Debug, Coding)]
pub struct Comment {
pub text: TStr,
pub location: Location,
}
#[derive(Clone, Debug, Coding)]
pub struct Member {
pub name: TStr,
pub exported: bool,
pub kind: MemberKind,
pub comments: Vec<TStr>,
}
#[derive(Clone, Debug, Coding)]
@@ -101,12 +77,54 @@ pub enum MemberKind {
#[derive(Clone, Debug, Coding)]
pub struct Module {
pub items: Vec<Item>,
pub members: Vec<Member>,
}
/// Evaluate a lazy member. This call will only be issued to each system once.
#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
#[extends(HostExtReq)]
pub struct GetMember(pub SysId, pub TreeId);
impl Request for GetMember {
type Response = MemberKind;
}
/// This request can only be issued while the interpreter is running, so during
/// an atom call.
#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
#[extends(ExtHostReq)]
pub struct LsModule(pub SysId, pub TStrv);
impl Request for LsModule {
type Response = Result<ModuleInfo, LsModuleError>;
}
#[derive(Clone, Debug, Coding)]
pub enum LsModuleError {
InvalidPath,
IsConstant,
TreeUnavailable,
}
/// Information about a module sent from the host to an extension. By necessity,
/// members and imports are non-overlapping.
#[derive(Clone, Debug, Coding)]
pub struct ModuleInfo {
/// List the names defined in this module
pub members: HashMap<TStr, MemberInfo>,
}
#[derive(Clone, Copy, Debug, Coding)]
pub struct MemberInfo {
/// true if the name is exported
pub public: bool,
/// Whether the tree item is a constant value or a module
pub kind: MemberInfoKind,
}
/// Indicates what kind of node a name refers to
#[derive(Clone, Copy, Debug, Coding)]
pub enum MemberInfoKind {
/// has children obtained with [crate::LsModule]
Module,
/// has a value retrievable in [crate::ExpressionKind::Const]
Constant,
}

47
orchid-api/src/vfs.rs
View File

@@ -1,47 +0,0 @@
use std::collections::HashMap;
use std::num::NonZeroU16;
use orchid_api_derive::{Coding, Hierarchy};
use orchid_api_traits::Request;
use crate::error::OrcResult;
use crate::interner::TStr;
use crate::proto::HostExtReq;
use crate::system::SysId;
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
pub struct VfsId(pub NonZeroU16);
#[derive(Clone, Debug, Coding)]
pub enum Loaded {
Code(String),
Collection(Vec<TStr>),
}
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(VfsReq, HostExtReq)]
pub struct VfsRead(pub SysId, pub VfsId, pub Vec<TStr>);
impl Request for VfsRead {
type Response = OrcResult<Loaded>;
}
#[derive(Clone, Debug, Coding)]
pub enum EagerVfs {
Lazy(VfsId),
Eager(HashMap<TStr, EagerVfs>),
}
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(VfsReq, HostExtReq)]
pub struct GetVfs(pub SysId);
impl Request for GetVfs {
type Response = EagerVfs;
}
#[derive(Clone, Debug, Coding, Hierarchy)]
#[extends(HostExtReq)]
#[extendable]
pub enum VfsReq {
GetVfs(GetVfs),
VfsRead(VfsRead),
}

5
orchid-base/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "orchid-base"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -20,10 +20,9 @@ num-traits = "0.2.19"
orchid-api = { version = "0.1.0", path = "../orchid-api" }
orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
ordered-float = "4.6.0"
ordered-float = "5.0.0"
regex = "1.11.1"
rust-embed = "8.5.0"
rust_decimal = "1.36.0"
some_executor = "0.4.0"
substack = "1.1.1"
test_executors = "0.3.2"

7
orchid-base/src/builtin.rs
View File

@@ -6,7 +6,6 @@ use futures::future::LocalBoxFuture;
use crate::api;
pub type Spawner = Rc<dyn Fn(LocalBoxFuture<'static, ()>)>;
pub type RecvCB<'a> = Box<dyn for<'b> FnOnce(&'b [u8]) -> LocalBoxFuture<'b, ()> + 'a>;
/// The 3 primary contact points with an extension are
/// - send a message
@@ -15,8 +14,10 @@ pub type RecvCB<'a> = Box<dyn for<'b> FnOnce(&'b [u8]) -> LocalBoxFuture<'b, ()>
///
/// There are no ordering guarantees about these
pub trait ExtPort {
#[must_use]
fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>;
fn recv<'a>(&'a self, cb: RecvCB<'a>) -> LocalBoxFuture<'a, ()>;
#[must_use]
fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>>;
}
pub struct ExtInit {
@@ -25,7 +26,7 @@ pub struct ExtInit {
}
impl ExtInit {
pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await }
pub async fn recv<'a, 's: 'a>(&'s self, cb: RecvCB<'a>) { self.port.recv(Box::new(cb)).await }
pub async fn recv(&self) -> Option<Vec<u8>> { self.port.recv().await }
}
impl Deref for ExtInit {
type Target = api::ExtensionHeader;

60
orchid-base/src/error.rs
View File

@@ -1,4 +1,5 @@
use std::cell::RefCell;
use std::ffi::OsStr;
use std::fmt;
use std::ops::Add;
use std::sync::Arc;
@@ -39,6 +40,14 @@ impl ErrPos {
impl From<Pos> for ErrPos {
fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
}
impl fmt::Display for ErrPos {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.message {
Some(msg) => write!(f, "{}: {}", self.position, msg),
None => write!(f, "{}", self.position),
}
}
}
#[derive(Clone, Debug)]
pub struct OrcErr {
@@ -71,7 +80,7 @@ impl From<OrcErr> for Vec<OrcErr> {
}
impl fmt::Display for OrcErr {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let pstr = self.positions.iter().map(|p| format!("{p:?}")).join("; ");
let pstr = self.positions.iter().map(|p| format!("{p}")).join("; ");
write!(f, "{}: {} @ {}", self.description, self.message, pstr)
}
}
@@ -161,28 +170,55 @@ pub fn mk_err(
}
}
pub fn mk_errv(
pub fn mk_errv<I: Into<ErrPos>>(
description: Tok<String>,
message: impl AsRef<str>,
posv: impl IntoIterator<Item = ErrPos>,
posv: impl IntoIterator<Item = I>,
) -> OrcErrv {
mk_err(description, message, posv).into()
mk_err(description, message, posv.into_iter().map_into()).into()
}
pub trait Reporter {
fn report(&self, e: impl Into<OrcErrv>);
pub async fn async_io_err<I: Into<ErrPos>>(
err: async_std::io::Error,
i: &Interner,
posv: impl IntoIterator<Item = I>,
) -> OrcErrv {
mk_errv(i.i(&err.kind().to_string()).await, err.to_string(), posv)
}
pub struct ReporterImpl {
pub async fn os_str_to_string<'a, I: Into<ErrPos>>(
str: &'a OsStr,
i: &Interner,
posv: impl IntoIterator<Item = I>,
) -> OrcRes<&'a str> {
match str.to_str() {
Some(str) => Ok(str),
None => Err(mk_errv(
i.i("Non-unicode string").await,
format!("{str:?} is not representable as unicode"),
posv,
)),
}
}
pub struct Reporter {
errors: RefCell<Vec<OrcErr>>,
}
impl ReporterImpl {
impl Reporter {
pub fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
pub fn new() -> Self { Self { errors: RefCell::new(vec![]) } }
pub fn errv(self) -> Option<OrcErrv> { OrcErrv::new(self.errors.into_inner()).ok() }
pub fn merge<T>(self, res: OrcRes<T>) -> OrcRes<T> {
match (res, self.errv()) {
(res, None) => res,
(Ok(_), Some(errv)) => Err(errv),
(Err(e), Some(errv)) => Err(e + errv),
}
}
pub fn is_empty(&self) -> bool { self.errors.borrow().is_empty() }
}
impl Reporter for ReporterImpl {
fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
}
impl Default for ReporterImpl {
impl Default for Reporter {
fn default() -> Self { Self::new() }
}

8
orchid-base/src/format.rs
View File

@@ -13,6 +13,7 @@ use crate::interner::Interner;
use crate::{api, match_mapping};
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
#[must_use]
pub struct FmtUnit {
pub subs: Vec<FmtUnit>,
pub variants: Rc<Variants>,
@@ -209,6 +210,9 @@ impl From<Rc<String>> for Variants {
impl From<String> for Variants {
fn from(value: String) -> Self { Self::from(Rc::new(value)) }
}
impl From<&str> for Variants {
fn from(value: &str) -> Self { Self::from(value.to_string()) }
}
impl FromStr for Variants {
type Err = Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(Self::default().bounded(s)) }
@@ -265,8 +269,12 @@ impl FmtCtx for FmtCtxImpl<'_> {
}
pub trait Format {
#[must_use]
fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> + 'a;
}
impl Format for Never {
async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit { match *self {} }
}
/// Format with default strategy. Currently equal to [take_first_fmt]
pub async fn fmt(v: &(impl Format + ?Sized), i: &Interner) -> String { take_first_fmt(v, i).await }

30
orchid-base/src/interner.rs
View File

@@ -98,7 +98,7 @@ impl Interned for String {
impl InternMarker for api::TStr {
type Interned = String;
async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
Tok::new(Rc::new(i.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
Tok::new(Rc::new(i.0.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
}
fn get_id(self) -> NonZeroU64 { self.0 }
fn from_id(id: NonZeroU64) -> Self { Self(id) }
@@ -125,7 +125,7 @@ impl Interned for Vec<Tok<String>> {
impl InternMarker for api::TStrv {
type Interned = Vec<Tok<String>>;
async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
let rep = i.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
let rep = i.0.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
let data = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
Tok::new(Rc::new(data), self)
}
@@ -217,24 +217,26 @@ pub struct TypedInterners {
}
#[derive(Default)]
pub struct Interner {
pub struct InternerData {
interners: Mutex<TypedInterners>,
master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
}
#[derive(Clone, Default)]
pub struct Interner(Rc<InternerData>);
impl Interner {
pub fn new_master() -> Self { Self::default() }
pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) -> Self {
Self { master: Some(Box::new(req)), interners: Mutex::default() }
Self(Rc::new(InternerData { master: Some(Box::new(req)), interners: Mutex::default() }))
}
/// Intern some data; query its identifier if not known locally
pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
let data = t.get_owned();
let mut g = self.interners.lock().await;
let mut g = self.0.interners.lock().await;
let typed = T::bimap(&mut g);
if let Some(tok) = typed.by_value(&data) {
return tok;
}
let marker = match &self.master {
let marker = match &self.0.master {
Some(c) => data.clone().intern(&**c).await,
None =>
T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
@@ -245,29 +247,29 @@ impl Interner {
}
/// Extern an identifier; query the data it represents if not known locally
pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
if let Some(tok) = M::Interned::bimap(&mut *self.interners.lock().await).by_marker(marker) {
if let Some(tok) = M::Interned::bimap(&mut *self.0.interners.lock().await).by_marker(marker) {
return tok;
}
assert!(self.master.is_some(), "ID not in local interner and this is master");
assert!(self.0.master.is_some(), "ID not in local interner and this is master");
let token = marker.resolve(self).await;
M::Interned::bimap(&mut *self.interners.lock().await).insert(token.clone());
M::Interned::bimap(&mut *self.0.interners.lock().await).insert(token.clone());
token
}
pub async fn sweep_replica(&self) -> api::Retained {
assert!(self.master.is_some(), "Not a replica");
let mut g = self.interners.lock().await;
assert!(self.0.master.is_some(), "Not a replica");
let mut g = self.0.interners.lock().await;
api::Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
}
pub async fn sweep_master(&self, retained: api::Retained) {
assert!(self.master.is_none(), "Not master");
let mut g = self.interners.lock().await;
assert!(self.0.master.is_none(), "Not master");
let mut g = self.0.interners.lock().await;
g.strings.sweep_master(retained.strings.into_iter().collect());
g.vecs.sweep_master(retained.vecs.into_iter().collect());
}
}
impl fmt::Debug for Interner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Interner{{ replica: {} }}", self.master.is_none())
write!(f, "Interner{{ replica: {} }}", self.0.master.is_none())
}
}

1
orchid-base/src/lib.rs
View File

@@ -15,7 +15,6 @@ pub mod interner;
pub mod join;
pub mod location;
pub mod logging;
pub mod macros;
mod match_mapping;
pub mod msg;
pub mod name;

49
orchid-base/src/location.rs
View File

@@ -6,6 +6,7 @@ use std::ops::Range;
use trait_set::trait_set;
use crate::error::ErrPos;
use crate::interner::{Interner, Tok};
use crate::name::Sym;
use crate::{api, match_mapping, sym};
@@ -23,15 +24,13 @@ pub enum Pos {
Inherit,
Gen(CodeGenInfo),
/// Range and file
SourceRange(SourceRange),
/// Range only, file implied. Most notably used by parsers
Range(Range<u32>),
SrcRange(SrcRange),
}
impl Pos {
pub fn pretty_print(&self, get_src: &mut impl GetSrc) -> String {
match self {
Self::Gen(g) => g.to_string(),
Self::SourceRange(sr) => sr.pretty_print(&get_src(&sr.path)),
Self::SrcRange(sr) => sr.pretty_print(&get_src(&sr.path)),
// Can't pretty print partial and meta-location
other => format!("{other:?}"),
}
@@ -39,30 +38,41 @@ impl Pos {
pub async fn from_api(api: &api::Location, i: &Interner) -> Self {
match_mapping!(api, api::Location => Pos {
None, Inherit, SlotTarget,
Range(r.clone()),
Gen(cgi => CodeGenInfo::from_api(cgi, i).await),
SourceRange(sr => SourceRange::from_api(sr, i).await)
} {
api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr, i).await)
})
}
pub fn to_api(&self) -> api::Location {
match_mapping!(self, Pos => api::Location {
None, Inherit, SlotTarget,
Range(r.clone()),
Gen(cgi.to_api()),
SourceRange(sr.to_api()),
} {
Self::SrcRange(sr) => api::Location::SourceRange(sr.to_api()),
})
}
}
impl fmt::Display for Pos {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Pos::Inherit => f.write_str("Unresolved inherited position"),
Pos::SlotTarget => f.write_str("Unresolved slot target position"),
Pos::None => f.write_str("N/A"),
Pos::Gen(g) => write!(f, "{g}"),
Pos::SrcRange(sr) => write!(f, "{sr}"),
}
}
}
/// Exact source code location. Includes where the code was loaded from, what
/// the original source code was, and a byte range.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct SourceRange {
pub struct SrcRange {
pub(crate) path: Sym,
pub(crate) range: Range<u32>,
}
impl SourceRange {
pub fn new(range: &Range<u32>, path: &Sym) -> Self {
impl SrcRange {
pub fn new(range: Range<u32>, path: &Sym) -> Self {
Self { range: range.clone(), path: path.clone() }
}
/// Create a dud [SourceRange] for testing. Its value is unspecified and
@@ -77,7 +87,7 @@ impl SourceRange {
/// 0-based index of last byte + 1
pub fn end(&self) -> u32 { self.range.end }
/// Syntactic location
pub fn pos(&self) -> Pos { Pos::SourceRange(self.clone()) }
pub fn pos(&self) -> Pos { Pos::SrcRange(self.clone()) }
/// Transform the numeric byte range
pub fn map_range(&self, map: impl FnOnce(Range<u32>) -> Range<u32>) -> Self {
Self { range: map(self.range()), path: self.path() }
@@ -92,13 +102,24 @@ impl SourceRange {
}
}
pub fn zw(path: Sym, pos: u32) -> Self { Self { path, range: pos..pos } }
async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
pub async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
Self { path: Sym::from_api(api.path, i).await, range: api.range.clone() }
}
fn to_api(&self) -> api::SourceRange {
pub fn to_api(&self) -> api::SourceRange {
api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
}
}
impl From<SrcRange> for ErrPos {
fn from(val: SrcRange) -> Self { val.pos().into() }
}
impl fmt::Display for SrcRange {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.range.len() {
0 => write!(f, "{}:{}", self.path(), self.range.start),
n => write!(f, "{}:{}+{}", self.path(), self.range.start, n),
}
}
}
/// Information about a code generator attached to the generated code
#[derive(Clone, PartialEq, Eq, Hash)]

168
orchid-base/src/macros.rs
View File

@@ -1,168 +0,0 @@
use std::marker::PhantomData;
use std::rc::Rc;
use std::sync::Arc;
use async_stream::stream;
use futures::future::{LocalBoxFuture, join_all};
use futures::{FutureExt, StreamExt};
use never::Never;
use trait_set::trait_set;
use crate::format::{FmtCtx, FmtUnit, Format, Variants};
use crate::interner::Interner;
use crate::location::Pos;
use crate::name::Sym;
use crate::tree::{Paren, Ph};
use crate::{api, match_mapping, tl_cache};
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
pub struct MacroSlot<'a>(api::MacroTreeId, PhantomData<&'a ()>);
impl MacroSlot<'_> {
pub fn id(self) -> api::MacroTreeId { self.0 }
}
trait_set! {
pub trait MacroAtomToApi<A> = for<'a> FnMut(&'a A) -> LocalBoxFuture<'a, api::MacroToken>;
pub trait MacroAtomFromApi<'a, A> =
for<'b> FnMut(&'b api::Atom) -> LocalBoxFuture<'b, MTok<'a, A>>;
}
#[derive(Clone, Debug)]
pub struct MTree<'a, A> {
pub pos: Pos,
pub tok: Rc<MTok<'a, A>>,
}
impl<'a, A> MTree<'a, A> {
pub(crate) async fn from_api(
api: &api::MacroTree,
do_atom: &mut impl MacroAtomFromApi<'a, A>,
i: &Interner,
) -> Self {
Self {
pos: Pos::from_api(&api.location, i).await,
tok: Rc::new(MTok::from_api(&api.token, i, do_atom).await),
}
}
pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroTree {
api::MacroTree {
location: self.pos.to_api(),
token: self.tok.to_api(do_atom).boxed_local().await,
}
}
}
impl<A: Format> Format for MTree<'_, A> {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
self.tok.print(c).boxed_local().await
}
}
#[derive(Clone, Debug)]
pub enum MTok<'a, A> {
S(Paren, Vec<MTree<'a, A>>),
Name(Sym),
Slot(MacroSlot<'a>),
Lambda(Vec<MTree<'a, A>>, Vec<MTree<'a, A>>),
Ph(Ph),
Atom(A),
/// Used in extensions to directly return input
Ref(Arc<MTok<'a, Never>>),
/// Used in the matcher to skip previous macro output which can only go in
/// vectorial placeholders
Done(Rc<MTok<'a, A>>),
}
impl<'a, A> MTok<'a, A> {
pub(crate) async fn from_api(
api: &api::MacroToken,
i: &Interner,
do_atom: &mut impl MacroAtomFromApi<'a, A>,
) -> Self {
match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
Lambda(x => mtreev_from_api(x, i, do_atom).await, b => mtreev_from_api(b, i, do_atom).await),
Name(t => Sym::from_api(*t, i).await),
Slot(tk => MacroSlot(*tk, PhantomData)),
S(p.clone(), b => mtreev_from_api(b, i, do_atom).await),
Ph(ph => Ph::from_api(ph, i).await),
} {
api::MacroToken::Atom(a) => do_atom(a).await
})
}
pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroToken {
fn sink<T>(n: &Never) -> LocalBoxFuture<'_, T> { match *n {} }
match_mapping!(&self, MTok => api::MacroToken {
Lambda(x => mtreev_to_api(x, do_atom).await, b => mtreev_to_api(b, do_atom).await),
Name(t.tok().to_api()),
Ph(ph.to_api()),
S(p.clone(), b => mtreev_to_api(b, do_atom).await),
Slot(tk.0.clone()),
} {
MTok::Ref(r) => r.to_api(&mut sink).boxed_local().await,
MTok::Done(t) => t.to_api(do_atom).boxed_local().await,
MTok::Atom(a) => do_atom(a).await,
})
}
pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Rc::new(self) } }
}
impl<A: Format> Format for MTok<'_, A> {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
match self {
Self::Atom(a) => a.print(c).await,
Self::Done(d) =>
FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("(Done){0l}"))), [
d.print(c).boxed_local().await,
]),
Self::Lambda(arg, b) => FmtUnit::new(
tl_cache!(Rc<Variants>: Rc::new(Variants::default()
.unbounded("\\{0b}.{1l}")
.bounded("(\\{0b}.{1b})"))),
[mtreev_fmt(arg, c).await, mtreev_fmt(b, c).await],
),
Self::Name(n) => format!("{n}").into(),
Self::Ph(ph) => format!("{ph}").into(),
Self::Ref(r) =>
FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("(ref){0l}"))), [
r.print(c).boxed_local().await,
]),
Self::S(p, body) => FmtUnit::new(
match *p {
Paren::Round => Rc::new(Variants::default().bounded("({0b})")),
Paren::Curly => Rc::new(Variants::default().bounded("{{0b}}")),
Paren::Square => Rc::new(Variants::default().bounded("[{0b}]")),
},
[mtreev_fmt(body, c).await],
),
Self::Slot(slot) => format!("{:?}", slot.0).into(),
}
}
}
pub async fn mtreev_from_api<'a, 'b, A>(
apiv: impl IntoIterator<Item = &'b api::MacroTree>,
i: &Interner,
do_atom: &'b mut (impl MacroAtomFromApi<'a, A> + 'b),
) -> Vec<MTree<'a, A>> {
stream! {
for api in apiv {
yield MTree::from_api(api, do_atom, i).boxed_local().await
}
}
.collect()
.await
}
pub async fn mtreev_to_api<'a: 'b, 'b, A: 'b>(
v: impl IntoIterator<Item = &'b MTree<'a, A>>,
do_atom: &mut impl MacroAtomToApi<A>,
) -> Vec<api::MacroTree> {
let mut out = Vec::new();
for t in v {
out.push(t.to_api(do_atom).await);
}
out
}
pub async fn mtreev_fmt<'a: 'b, 'b, A: 'b + Format>(
v: impl IntoIterator<Item = &'b MTree<'a, A>>,
c: &(impl FmtCtx + ?Sized),
) -> FmtUnit {
FmtUnit::sequence(" ", None, join_all(v.into_iter().map(|t| t.print(c))).await)
}

12
orchid-base/src/name.rs
View File

@@ -2,11 +2,10 @@
use std::borrow::Borrow;
use std::hash::Hash;
use std::iter::Cloned;
use std::num::{NonZeroU64, NonZeroUsize};
use std::ops::{Bound, Deref, Index, RangeBounds};
use std::ops::{Deref, Index};
use std::path::Path;
use std::{fmt, slice, vec};
use std::{fmt, vec};
use futures::future::{OptionFuture, join_all};
use itertools::Itertools;
@@ -54,12 +53,12 @@ impl VPath {
pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
/// Add a token to the path. Since now we know that it can't be empty, turn it
/// into a name.
pub fn name_with_prefix(self, name: Tok<String>) -> VName {
pub fn name_with_suffix(self, name: Tok<String>) -> VName {
VName(self.into_iter().chain([name]).collect())
}
/// Add a token to the beginning of the. Since now we know that it can't be
/// empty, turn it into a name.
pub fn name_with_suffix(self, name: Tok<String>) -> VName {
pub fn name_with_prefix(self, name: Tok<String>) -> VName {
VName([name].into_iter().chain(self).collect())
}
@@ -237,6 +236,9 @@ impl Sym {
Self::from_tok(Tok::from_api(marker, i).await).expect("Empty sequence found for serialized Sym")
}
pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
pub async fn push(&self, tok: Tok<String>, i: &Interner) -> Sym {
Self::new(self.0.iter().cloned().chain([tok]), i).await.unwrap()
}
}
impl fmt::Debug for Sym {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Sym({self})") }

75
orchid-base/src/number.rs
View File

@@ -1,35 +1,27 @@
use std::num::IntErrorKind;
use std::ops::Range;
use num_traits::ToPrimitive;
use ordered_float::NotNan;
use rust_decimal::Decimal;
use crate::error::{OrcErr, mk_err};
use crate::interner::Interner;
use crate::location::Pos;
use crate::location::SrcRange;
use crate::name::Sym;
/// A number, either floating point or unsigned int, parsed by Orchid.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Numeric {
/// A nonnegative integer
Uint(u64),
/// An integer
Int(i64),
/// A binary float other than NaN
Float(NotNan<f64>),
/// A decimal number
Decimal(Decimal),
}
impl Numeric {
pub fn decimal(num: i64, scale: u32) -> Self { Self::Decimal(Decimal::new(num, scale)) }
pub fn float(value: f64) -> Self { Self::Float(NotNan::new(value).unwrap()) }
pub fn to_f64(self) -> NotNan<f64> {
match self {
Self::Float(f) => f,
Self::Decimal(d) => {
let f = d.to_f64().expect("This is apparently always possible");
NotNan::new(f).expect("decimal was nan")
},
Self::Uint(i) => NotNan::new(i as f64).expect("int cannot be NaN"),
Self::Int(i) => NotNan::new(i as f64).expect("int cannot be NaN"),
}
}
}
@@ -63,7 +55,12 @@ pub struct NumError {
pub kind: NumErrorKind,
}
pub async fn num_to_err(NumError { kind, range }: NumError, offset: u32, i: &Interner) -> OrcErr {
pub async fn num_to_err(
NumError { kind, range }: NumError,
offset: u32,
source: &Sym,
i: &Interner,
) -> OrcErr {
mk_err(
i.i("Failed to parse number").await,
match kind {
@@ -71,37 +68,37 @@ pub async fn num_to_err(NumError { kind, range }: NumError, offset: u32, i: &Int
NumErrorKind::InvalidDigit => "non-digit character encountered",
NumErrorKind::Overflow => "The number being described is too large or too accurate",
},
[Pos::Range(offset + range.start as u32..offset + range.end as u32).into()],
[SrcRange::new(offset + range.start as u32..offset + range.end as u32, source).pos().into()],
)
}
/// Parse a numbre literal out of text
pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
let overflow_err = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };
let overflow_e = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };
let (radix, noprefix, pos) = (string.strip_prefix("0x").map(|s| (16u8, s, 2)))
.or_else(|| string.strip_prefix("0b").map(|s| (2u8, s, 2)))
.or_else(|| string.strip_prefix("0o").map(|s| (8u8, s, 2)))
.unwrap_or((10u8, string, 0));
eprintln!("({radix}, {noprefix}, {pos})");
// identity
let (base, exponent) = match noprefix.split_once('p') {
let (base_s, exponent) = match noprefix.split_once('p') {
Some((b, e)) => {
let (s, d, len) = e.strip_prefix('-').map_or((1, e, 0), |ue| (-1, ue, 1));
(b, s * int_parse(d, 10, pos + b.len() + 1 + len)? as i32)
},
None => (noprefix, 0),
};
eprintln!("({base},{exponent})");
match base.split_once('.') {
eprintln!("({base_s},{exponent})");
match base_s.split_once('.') {
None => {
let base_usize = int_parse(base, radix, pos)?;
let base = int_parse(base_s, radix, pos)?;
if let Ok(pos_exp) = u32::try_from(exponent) {
if let Some(radical) = u64::from(radix).checked_pow(pos_exp) {
let number = base_usize.checked_mul(radical).ok_or(overflow_err)?;
return Ok(Numeric::Uint(number));
let num = base.checked_mul(radical).and_then(|m| m.try_into().ok()).ok_or(overflow_e)?;
return Ok(Numeric::Int(num));
}
}
let f = (base_usize as f64) * (radix as f64).powi(exponent);
let f = (base as f64) * (radix as f64).powi(exponent);
let err = NumError { range: 0..string.len(), kind: NumErrorKind::NaN };
Ok(Numeric::Float(NotNan::new(f).map_err(|_| err)?))
},
@@ -109,25 +106,9 @@ pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
let whole_n = int_parse(whole, radix, pos)?;
let part_n = int_parse(part, radix, pos + whole.len() + 1)?;
let scale = part.chars().filter(|c| *c != '_').count() as u32;
if radix == 10 {
let scaled_unit = 10u64.checked_pow(scale).ok_or(overflow_err.clone())?;
let scaled_n = i128::from(whole_n) * i128::from(scaled_unit) + i128::from(part_n);
let decimal = Decimal::from_i128_with_scale(scaled_n, scale);
let p = if let Ok(uexp) = u32::try_from(exponent) {
let e_multiplier = 10i64.checked_pow(uexp).ok_or(overflow_err)?;
Decimal::new(e_multiplier, 0)
} else {
let inv_oom = u32::try_from(-exponent).map_err(|_| overflow_err)?;
eprintln!("inv_oom: {inv_oom}");
Decimal::new(1, inv_oom)
};
eprintln!("({scaled_n}, {scale}, {p})");
Ok(Numeric::Decimal(decimal * p))
} else {
let real_val = whole_n as f64 + (part_n as f64 / (radix as f64).powi(scale as i32));
let f = real_val * (radix as f64).powi(exponent);
Ok(Numeric::Float(NotNan::new(f).expect("None of the inputs are NaN")))
}
let real_val = whole_n as f64 + (part_n as f64 / (radix as f64).powi(scale as i32));
let f = real_val * (radix as f64).powi(exponent);
Ok(Numeric::Float(NotNan::new(f).expect("None of the inputs are NaN")))
},
}
}
@@ -168,7 +149,7 @@ mod test {
#[test]
fn just_ints() {
let test = |s, n| assert_eq!(parse_num(s), Ok(Numeric::Uint(n)));
let test = |s, n| assert_eq!(parse_num(s), Ok(Numeric::Int(n)));
test("12345", 12345);
test("0xcafebabe", 0xcafebabe);
test("0o751", 0o751);
@@ -178,11 +159,11 @@ mod test {
#[test]
fn decimals() {
let test = |s, n| assert_eq!(parse_num(s), Ok(n));
test("3.1417", Numeric::decimal(31417, 4));
test("3.1417", Numeric::float(3.1417));
test("0xf.cafe", Numeric::float(0xf as f64 + 0xcafe as f64 / 0x10000 as f64));
test("34p3", Numeric::Uint(34000));
test("0x2p3", Numeric::Uint(0x2 * 0x1000));
test("1.5p3", Numeric::decimal(1500, 0));
test("34p3", Numeric::Int(34000));
test("0x2p3", Numeric::Int(0x2 * 0x1000));
test("1.5p3", Numeric::float(1500.0));
test("0x2.5p3", Numeric::float((0x25 * 0x100) as f64));
}
}

350
orchid-base/src/parse.rs
View File

@@ -1,17 +1,33 @@
use std::fmt::{self, Display};
use std::iter;
use std::ops::{Deref, Range};
use std::ops::Deref;
use futures::FutureExt;
use futures::future::join_all;
use itertools::Itertools;
use crate::api;
use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
use crate::format::{Format, take_first_fmt};
use crate::format::fmt;
use crate::interner::{Interner, Tok};
use crate::location::Pos;
use crate::name::{VName, VPath};
use crate::tree::{AtomRepr, ExtraTok, Paren, TokTree, Token};
use crate::location::SrcRange;
use crate::name::{Sym, VName, VPath};
use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_range};
pub trait ParseCtx {
#[must_use]
fn i(&self) -> &Interner;
#[must_use]
fn reporter(&self) -> &Reporter;
}
pub struct ParseCtxImpl<'a> {
pub i: &'a Interner,
pub r: &'a Reporter,
}
impl ParseCtx for ParseCtxImpl<'_> {
fn i(&self) -> &Interner { self.i }
fn reporter(&self) -> &Reporter { self.r }
}
pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
@@ -21,51 +37,41 @@ pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
/// A cheaply copiable subsection of a document that holds onto context data and
/// one token for error reporting on empty subsections.
#[derive(Debug)]
pub struct Snippet<'a, 'b, A: AtomRepr, X: ExtraTok> {
prev: &'a TokTree<'b, A, X>,
cur: &'a [TokTree<'b, A, X>],
interner: &'a Interner,
pub struct Snippet<'a, A: ExprRepr, X: ExtraTok> {
prev: &'a TokTree<A, X>,
cur: &'a [TokTree<A, X>],
}
impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
pub fn new(
prev: &'a TokTree<'b, A, X>,
cur: &'a [TokTree<'b, A, X>],
interner: &'a Interner,
) -> Self {
Self { prev, cur, interner }
}
pub fn i(&self) -> &'a Interner { self.interner }
impl<'a, A, X> Snippet<'a, A, X>
where
A: ExprRepr,
X: ExtraTok,
{
pub fn new(prev: &'a TokTree<A, X>, cur: &'a [TokTree<A, X>]) -> Self { Self { prev, cur } }
pub fn split_at(self, pos: u32) -> (Self, Self) {
let Self { prev, cur, interner } = self;
let fst = Self { prev, cur: &cur[..pos as usize], interner };
let Self { prev, cur } = self;
let fst = Self { prev, cur: &cur[..pos as usize] };
let new_prev = if pos == 0 { self.prev } else { &self.cur[pos as usize - 1] };
let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..], interner };
let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..] };
(fst, snd)
}
pub fn find_idx(self, mut f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<u32> {
pub fn find_idx(self, mut f: impl FnMut(&Token<A, X>) -> bool) -> Option<u32> {
self.cur.iter().position(|t| f(&t.tok)).map(|t| t as u32)
}
pub fn get(self, idx: u32) -> Option<&'a TokTree<'b, A, X>> { self.cur.get(idx as usize) }
pub fn get(self, idx: u32) -> Option<&'a TokTree<A, X>> { self.cur.get(idx as usize) }
pub fn len(self) -> u32 { self.cur.len() as u32 }
pub fn prev(self) -> &'a TokTree<'b, A, X> { self.prev }
pub fn pos(self) -> Range<u32> {
(self.cur.first().map(|f| f.range.start..self.cur.last().unwrap().range.end))
.unwrap_or(self.prev.range.clone())
}
pub fn pop_front(self) -> Option<(&'a TokTree<'b, A, X>, Self)> {
pub fn prev(self) -> &'a TokTree<A, X> { self.prev }
pub fn sr(self) -> SrcRange { ttv_range(self.cur).unwrap_or_else(|| self.prev.sr.clone()) }
pub fn pop_front(self) -> Option<(&'a TokTree<A, X>, Self)> {
self.cur.first().map(|r| (r, self.split_at(1).1))
}
pub fn pop_back(self) -> Option<(Self, &'a TokTree<'b, A, X>)> {
pub fn pop_back(self) -> Option<(Self, &'a TokTree<A, X>)> {
self.cur.last().map(|r| (self.split_at(self.len() - 1).0, r))
}
pub fn split_once(self, f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<(Self, Self)> {
pub fn split_once(self, f: impl FnMut(&Token<A, X>) -> bool) -> Option<(Self, Self)> {
let idx = self.find_idx(f)?;
Some((self.split_at(idx).0, self.split_at(idx + 1).1))
}
pub fn split(
mut self,
mut f: impl FnMut(&Token<'b, A, X>) -> bool,
) -> impl Iterator<Item = Self> {
pub fn split(mut self, mut f: impl FnMut(&Token<A, X>) -> bool) -> impl Iterator<Item = Self> {
iter::from_fn(move || {
if self.is_empty() {
return None;
@@ -77,26 +83,22 @@ impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
}
pub fn is_empty(self) -> bool { self.len() == 0 }
pub fn skip_fluff(self) -> Self {
let non_fluff_start = self.find_idx(|t| !matches!(t, Token::NS | Token::Comment(_)));
let non_fluff_start = self.find_idx(|t| !matches!(t, Token::BR | Token::Comment(_)));
self.split_at(non_fluff_start.unwrap_or(self.len())).1
}
/// Format the argument using the context held in this snippet
pub async fn fmt(self, v: &(impl Format + ?Sized)) -> String { take_first_fmt(v, self.i()).await }
}
impl<A: AtomRepr, X: ExtraTok> Copy for Snippet<'_, '_, A, X> {}
impl<A: AtomRepr, X: ExtraTok> Clone for Snippet<'_, '_, A, X> {
impl<A: ExprRepr, X: ExtraTok> Copy for Snippet<'_, A, X> {}
impl<A: ExprRepr, X: ExtraTok> Clone for Snippet<'_, A, X> {
fn clone(&self) -> Self { *self }
}
impl<'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'_, 'b, A, X> {
type Target = [TokTree<'b, A, X>];
impl<A: ExprRepr, X: ExtraTok> Deref for Snippet<'_, A, X> {
type Target = [TokTree<A, X>];
fn deref(&self) -> &Self::Target { self.cur }
}
/// Remove tokens that aren't meaningful in expression context, such as comments
/// or line breaks
pub fn strip_fluff<'a, A: AtomRepr, X: ExtraTok>(
tt: &TokTree<'a, A, X>,
) -> Option<TokTree<'a, A, X>> {
pub fn strip_fluff<A: ExprRepr, X: ExtraTok>(tt: &TokTree<A, X>) -> Option<TokTree<A, X>> {
let tok = match &tt.tok {
Token::BR => return None,
Token::Comment(_) => return None,
@@ -104,20 +106,30 @@ pub fn strip_fluff<'a, A: AtomRepr, X: ExtraTok>(
Token::S(p, b) => Token::S(*p, b.iter().filter_map(strip_fluff).collect()),
t => t.clone(),
};
Some(TokTree { tok, range: tt.range.clone() })
Some(TokTree { tok, sr: tt.sr.clone() })
}
#[derive(Clone, Debug)]
pub struct Comment {
pub text: Tok<String>,
pub pos: Pos,
pub sr: SrcRange,
}
impl Comment {
pub fn to_api(&self) -> api::Comment {
api::Comment { location: self.pos.to_api(), text: self.text.to_api() }
// XXX: which of these four are actually used?
pub async fn from_api(c: &api::Comment, src: Sym, i: &Interner) -> Self {
Self { text: i.ex(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
}
pub async fn from_api(api: &api::Comment, i: &Interner) -> Self {
Self { pos: Pos::from_api(&api.location, i).await, text: Tok::from_api(api.text, i).await }
pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, i: &Interner) -> Option<Self> {
match &tk.tok {
Token::Comment(text) => Some(Self { text: i.i(&**text).await, sr: tk.sr.clone() }),
_ => None,
}
}
pub fn to_tk<R: ExprRepr, X: ExtraTok>(&self) -> TokTree<R, X> {
TokTree { tok: Token::Comment(self.text.rc().clone()), sr: self.sr.clone() }
}
pub fn to_api(&self) -> api::Comment {
api::Comment { range: self.sr.range(), text: self.text.to_api() }
}
}
@@ -125,9 +137,10 @@ impl fmt::Display for Comment {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "--[{}]--", self.text) }
}
pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
snip: Snippet<'a, 'b, A, X>,
) -> Vec<Parsed<'a, 'b, Vec<Comment>, A, X>> {
pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
ctx: &impl ParseCtx,
snip: Snippet<'a, A, X>,
) -> Vec<Parsed<'a, Vec<Comment>, A, X>> {
let mut items = Vec::new();
let mut comments = Vec::new();
for mut line in snip.split(|t| matches!(t, Token::BR)) {
@@ -141,11 +154,7 @@ pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
Some(i) => {
let (cmts, tail) = line.split_at(i);
let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
match &t.tok {
Token::Comment(c) =>
Comment { text: tail.i().i(&**c).await, pos: Pos::Range(t.range.clone()) },
_ => unreachable!("All are comments checked above"),
}
Comment::from_tk(t, ctx.i()).await.expect("All are comments checked above")
}))
.await;
items.push(Parsed { output: comments, tail });
@@ -155,166 +164,138 @@ pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
items
}
pub async fn try_pop_no_fluff<'a, 'b, A: AtomRepr, X: ExtraTok>(
snip: Snippet<'a, 'b, A, X>,
) -> ParseRes<'a, 'b, &'a TokTree<'b, A, X>, A, X> {
pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
ctx: &impl ParseCtx,
snip: Snippet<'a, A, X>,
) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
match snip.skip_fluff().pop_front() {
Some((output, tail)) => Ok(Parsed { output, tail }),
None =>
Err(mk_errv(snip.i().i("Unexpected end").await, "Pattern ends abruptly", [Pos::Range(
snip.pos(),
)
.into()])),
None => Err(mk_errv(
ctx.i().i("Unexpected end").await,
"Line ends abruptly; more tokens were expected",
[snip.sr()],
)),
}
}
pub async fn expect_end(snip: Snippet<'_, '_, impl AtomRepr, impl ExtraTok>) -> OrcRes<()> {
pub async fn expect_end(
ctx: &impl ParseCtx,
snip: Snippet<'_, impl ExprRepr, impl ExtraTok>,
) -> OrcRes<()> {
match snip.skip_fluff().get(0) {
Some(surplus) => Err(mk_errv(
snip.i().i("Extra code after end of line").await,
ctx.i().i("Extra code after end of line").await,
"Code found after the end of the line",
[Pos::Range(surplus.range.clone()).into()],
[surplus.sr.pos()],
)),
None => Ok(()),
}
}
pub async fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>(
snip: Snippet<'a, 'b, A, X>,
pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
ctx: &impl ParseCtx,
snip: Snippet<'a, A, X>,
tok: Tok<String>,
) -> ParseRes<'a, 'b, (), A, X> {
let Parsed { output: head, tail } = try_pop_no_fluff(snip).await?;
) -> ParseRes<'a, (), A, X> {
let Parsed { output: head, tail } = try_pop_no_fluff(ctx, snip).await?;
match &head.tok {
Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
t => Err(mk_errv(
snip.i().i("Expected specific keyword").await,
format!("Expected {tok} but found {:?}", snip.fmt(t).await),
[Pos::Range(head.range.clone()).into()],
ctx.i().i("Expected specific keyword").await,
format!("Expected {tok} but found {:?}", fmt(t, ctx.i()).await),
[head.sr()],
)),
}
}
pub struct Parsed<'a, 'b, T, A: AtomRepr, X: ExtraTok> {
pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
pub output: T,
pub tail: Snippet<'a, 'b, A, X>,
pub tail: Snippet<'a, H, X>,
}
pub type ParseRes<'a, 'b, T, A, X> = OrcRes<Parsed<'a, 'b, T, A, X>>;
pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
pub async fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
ctx: &(impl Reporter + ?Sized),
tail: Snippet<'a, 'b, A, X>,
) -> ParseRes<'a, 'b, Vec<(Import, Pos)>, A, X> {
let ret = rec(ctx, tail).await;
pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
ctx: &impl ParseCtx,
tail: Snippet<'a, A, X>,
) -> ParseRes<'a, Vec<Import>, A, X> {
let Some((tt, tail)) = tail.skip_fluff().pop_front() else {
return Err(mk_errv(
ctx.i().i("Expected token").await,
"Expected a name, a parenthesized list of names, or a globstar.",
[tail.sr().pos()],
));
};
let ret = rec(tt, ctx).await;
#[allow(clippy::type_complexity)] // it's an internal function
pub async fn rec<'a, 'b, A: AtomRepr, X: ExtraTok>(
ctx: &(impl Reporter + ?Sized),
tail: Snippet<'a, 'b, A, X>,
) -> ParseRes<'a, 'b, Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, A, X> {
let comma = tail.i().i(",").await;
let globstar = tail.i().i("*").await;
let Some((name, tail)) = tail.skip_fluff().pop_front() else {
return Err(mk_errv(
tail.i().i("Expected name").await,
"Expected a name, a list of names, or a globstar.",
[Pos::Range(tail.pos()).into()],
));
};
if let Some((Token::NS, tail)) = tail.skip_fluff().pop_front().map(|(tt, s)| (&tt.tok, s)) {
let n = match &name.tok {
Token::Name(n) if n.starts_with(name_start) => Ok(n),
_ =>
Err(mk_err(tail.i().i("Unexpected name prefix").await, "Only names can precede ::", [
Pos::Range(name.range.clone()).into(),
])),
};
match (Box::pin(rec(ctx, tail)).await, n) {
(Err(ev), n) => Err(ev.extended(n.err())),
(Ok(Parsed { tail, .. }), Err(e)) => {
ctx.report(e);
Ok(Parsed { output: vec![], tail })
},
(Ok(Parsed { tail, output }), Ok(pre)) => Ok(Parsed {
output: output.into_iter().update(|i| i.0.push(pre.clone())).collect_vec(),
tail,
}),
}
} else {
let output = match &name.tok {
Token::Name(ntok) => {
let nopt = match ntok {
n if *n == globstar => None,
n if n.starts_with(op_char) => {
return Err(mk_errv(
tail.i().i("Unescaped operator in multiname").await,
"Operators in multinames should be enclosed in []",
[Pos::Range(name.range.clone()).into()],
));
},
n => Some(n.clone()),
};
vec![(vec![], nopt, Pos::Range(name.range.clone()))]
},
Token::S(Paren::Square, b) => {
let mut ok = Vec::new();
for tt in b.iter() {
match &tt.tok {
Token::Name(n) if n.starts_with(op_char) =>
ok.push((vec![], Some(n.clone()), Pos::Range(tt.range.clone()))),
Token::BR | Token::Comment(_) => (),
_ => ctx.report(mk_err(
tail.i().i("Non-operator in escapement in multiname").await,
"In multinames, [] functions as a literal name list reserved for operators",
[Pos::Range(name.range.clone()).into()],
)),
}
pub async fn rec<A: ExprRepr, X: ExtraTok>(
tt: &TokTree<A, X>,
ctx: &impl ParseCtx,
) -> OrcRes<Vec<(Vec<Tok<String>>, Option<Tok<String>>, SrcRange)>> {
let ttpos = tt.sr.pos();
match &tt.tok {
Token::NS(ns, body) => {
if !ns.starts_with(name_start) {
ctx.reporter().report(mk_err(
ctx.i().i("Unexpected name prefix").await,
"Only names can precede ::",
[ttpos.into()],
))
};
let out = Box::pin(rec(body, ctx)).await?;
Ok(out.into_iter().update(|i| i.0.push(ns.clone())).collect_vec())
},
Token::Name(ntok) => {
let n = ntok;
let nopt = Some(n.clone());
Ok(vec![(vec![], nopt, tt.sr.clone())])
},
Token::S(Paren::Round, b) => {
let mut o = Vec::new();
let mut body = Snippet::new(tt, b);
while let Some((output, tail)) = body.pop_front() {
match rec(output, ctx).boxed_local().await {
Ok(names) => o.extend(names),
Err(e) => ctx.reporter().report(e),
}
ok
},
Token::S(Paren::Round, b) => {
let mut ok = Vec::new();
let body = Snippet::new(name, b, tail.interner);
for csent in body.split(|n| matches!(n, Token::Name(n) if *n == comma)) {
match Box::pin(rec(ctx, csent)).await {
Err(e) => ctx.report(e),
Ok(Parsed { output, tail }) => match tail.get(0) {
None => ok.extend(output),
Some(t) => ctx.report(mk_err(
tail.i().i("Unexpected token in multiname group").await,
"Unexpected token. Likely missing a :: or , or wanted [] instead of ()",
[Pos::Range(t.range.clone()).into()],
)),
},
}
}
ok
},
t => {
return Err(mk_errv(
tail.i().i("Unrecognized name end").await,
format!("Names cannot end with {:?} tokens", tail.fmt(t).await),
[Pos::Range(name.range.clone()).into()],
));
},
};
Ok(Parsed { output, tail })
body = tail;
}
Ok(o)
},
t => {
return Err(mk_errv(
ctx.i().i("Unrecognized name end").await,
format!("Names cannot end with {:?} tokens", fmt(t, ctx.i()).await),
[ttpos],
));
},
}
}
ret.map(|Parsed { output, tail }| {
ret.map(|output| {
let output = (output.into_iter())
.map(|(p, name, pos)| (Import { path: VPath::new(p.into_iter().rev()), name }, pos))
.map(|(p, name, sr)| Import { path: VPath::new(p.into_iter().rev()), name, sr })
.collect_vec();
Parsed { output, tail }
})
}
/// A compound name, possibly ending with a globstar
/// A compound name, possibly ending with a globstar. It cannot be just a
/// globstar; either the name has to be known or the path has to be non-empty.
#[derive(Debug, Clone)]
pub struct Import {
pub path: VPath,
pub name: Option<Tok<String>>,
pub sr: SrcRange,
}
impl Import {
/// Most specific concrete path
pub fn mspath(self) -> VName {
match self.name {
Some(n) => self.path.name_with_suffix(n),
None => self.path.into_name().expect("Import cannot be empty"),
}
}
}
impl Display for Import {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str()))
@@ -327,14 +308,5 @@ mod test {
use super::Snippet;
fn _covary_snip_a<'a, 'b>(
x: Snippet<'static, 'b, Never, Never>,
) -> Snippet<'a, 'b, Never, Never> {
x
}
fn _covary_snip_b<'a, 'b>(
x: Snippet<'a, 'static, Never, Never>,
) -> Snippet<'a, 'b, Never, Never> {
x
}
fn _covary_snip_a<'a>(x: Snippet<'static, Never, Never>) -> Snippet<'a, Never, Never> { x }
}

21
orchid-base/src/reqnot.rs
View File

@@ -1,4 +1,4 @@
use std::any::{Any, TypeId};
use std::any::Any;
use std::cell::RefCell;
use std::future::Future;
use std::marker::PhantomData;
@@ -17,8 +17,8 @@ use hashbrown::HashMap;
use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
use trait_set::trait_set;
use crate::clone;
use crate::logging::Logger;
use crate::{api, clone};
pub struct Receipt<'a>(PhantomData<&'a mut ()>);
@@ -46,6 +46,9 @@ pub trait ReqHandlish {
}
fn defer_drop_objsafe(&self, val: Box<dyn Any>);
}
impl ReqHandlish for &'_ dyn ReqHandlish {
fn defer_drop_objsafe(&self, val: Box<dyn Any>) { (**self).defer_drop_objsafe(val) }
}
#[derive(destructure)]
pub struct RequestHandle<'a, MS: MsgSet> {
@@ -170,12 +173,16 @@ pub trait DynRequester {
pub struct MappedRequester<'a, T: 'a>(Box<dyn Fn(T) -> LocalBoxFuture<'a, RawReply> + 'a>, Logger);
impl<'a, T> MappedRequester<'a, T> {
fn new<U: DynRequester + 'a>(req: U, logger: Logger) -> Self
where T: Into<U::Transfer> {
fn new<U: DynRequester + 'a, F: Fn(T) -> U::Transfer + 'a>(
req: U,
cb: F,
logger: Logger,
) -> Self {
let req_arc = Arc::new(req);
let cb_arc = Arc::new(cb);
MappedRequester(
Box::new(move |t| {
Box::pin(clone!(req_arc; async move { req_arc.raw_request(t.into()).await}))
Box::pin(clone!(req_arc, cb_arc; async move { req_arc.raw_request(cb_arc(t)).await}))
}),
logger,
)
@@ -217,10 +224,10 @@ pub trait Requester: DynRequester {
&self,
data: R,
) -> impl Future<Output = R::Response>;
fn map<'a, U: Into<Self::Transfer>>(self) -> MappedRequester<'a, U>
fn map<'a, U>(self, cb: impl Fn(U) -> Self::Transfer + 'a) -> MappedRequester<'a, U>
where Self: Sized + 'a {
let logger = self.logger().clone();
MappedRequester::new(self, logger)
MappedRequester::new(self, cb, logger)
}
}

326
orchid-base/src/tree.rs
View File

@@ -2,54 +2,79 @@ use std::borrow::Borrow;
use std::fmt::{self, Debug, Display};
use std::future::Future;
use std::marker::PhantomData;
use std::ops::Range;
use std::rc::Rc;
use std::sync::Arc;
pub use api::PhKind;
use async_stream::stream;
use futures::future::{LocalBoxFuture, join_all};
use futures::future::join_all;
use futures::{FutureExt, StreamExt};
use itertools::Itertools;
use never::Never;
use ordered_float::NotNan;
use orchid_api_traits::Coding;
use trait_set::trait_set;
use crate::error::OrcErrv;
use crate::format::{FmtCtx, FmtUnit, Format, Variants};
use crate::interner::{Interner, Tok};
use crate::location::Pos;
use crate::location::{Pos, SrcRange};
use crate::name::Sym;
use crate::parse::Snippet;
use crate::{api, match_mapping, tl_cache};
trait_set! {
pub trait RecurCB<'a, A: AtomRepr, X: ExtraTok> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
pub trait ExtraTok = Format + Clone + fmt::Debug;
pub trait RefDoExtra<X> =
for<'b> FnMut(&'b X, Range<u32>) -> LocalBoxFuture<'b, api::TokenTree>;
pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::Debug {
type FromApiCtx<'a>;
type ToApiCtx<'a>;
#[must_use]
fn from_api(
api: &ApiEquiv,
ctx: &mut Self::FromApiCtx<'_>,
pos: SrcRange,
i: &Interner,
) -> impl Future<Output = Self>;
#[must_use]
fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>;
}
impl<T: Clone + Debug + Coding> TokenVariant<T> for Never {
type FromApiCtx<'a> = ();
type ToApiCtx<'a> = ();
async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange, _: &Interner) -> Self {
panic!("Cannot deserialize Never")
}
async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} }
}
pub fn recur<'a, A: AtomRepr, X: ExtraTok>(
tt: TokTree<'a, A, X>,
f: &impl Fn(TokTree<'a, A, X>, &dyn RecurCB<'a, A, X>) -> TokTree<'a, A, X>,
) -> TokTree<'a, A, X> {
f(tt, &|TokTree { range, tok }| {
trait_set! {
// TokenHandle
pub trait ExprRepr = TokenVariant<api::ExprTicket>;
// TokenExpr
pub trait ExtraTok = TokenVariant<api::Expression>;
}
trait_set! {
pub trait RecurCB<H: ExprRepr, X: ExtraTok> = Fn(TokTree<H, X>) -> TokTree<H, X>;
}
pub fn recur<H: ExprRepr, X: ExtraTok>(
tt: TokTree<H, X>,
f: &impl Fn(TokTree<H, X>, &dyn RecurCB<H, X>) -> TokTree<H, X>,
) -> TokTree<H, X> {
f(tt, &|TokTree { sr: range, tok }| {
let tok = match tok {
tok @ (Token::Atom(_) | Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::NS) => tok,
tok @ (Token::Name(_) | Token::Slot(_) | Token::X(_) | Token::Ph(_) | Token::Macro(_)) => tok,
tok @ Token::Reference(_) => tok,
tok @ (Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::Name(_)) => tok,
tok @ (Token::Handle(_) | Token::NewExpr(_)) => tok,
Token::NS(n, b) => Token::NS(n, Box::new(recur(*b, f))),
Token::LambdaHead(arg) =>
Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
};
TokTree { range, tok }
TokTree { sr: range, tok }
})
}
pub trait AtomRepr: Clone + Format {
type Ctx: ?Sized;
#[must_use]
fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> impl Future<Output = Self>;
#[must_use]
fn to_api(&self) -> impl Future<Output = orchid_api::Atom> + '_;
}
impl AtomRepr for Never {
@@ -71,139 +96,125 @@ impl Display for TokHandle<'_> {
}
#[derive(Clone, Debug)]
pub struct TokTree<'a, A: AtomRepr, X: ExtraTok> {
pub tok: Token<'a, A, X>,
pub range: Range<u32>,
pub struct TokTree<H: ExprRepr, X: ExtraTok> {
pub tok: Token<H, X>,
/// The protocol has a Range<u32> because these are always transmitted in the
/// context of a given snippet, but internal logic and error reporting is
/// easier if the in-memory representation also includes the snippet path.
pub sr: SrcRange,
}
impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
pub async fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx, i: &Interner) -> Self {
let tok = match_mapping!(&tt.token, api::Token => Token::<'b, A, X> {
BR, NS,
Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx).await),
Bottom(e => OrcErrv::from_api(e, i).await),
LambdaHead(arg => ttv_from_api(arg, ctx, i).await),
Name(n => Tok::from_api(*n, i).await),
S(*par, b => ttv_from_api(b, ctx, i).await),
Comment(c.clone()),
Slot(id => TokHandle::new(*id)),
Ph(ph => Ph::from_api(ph, i).await),
Macro(*prio),
Reference(tok => Sym::from_api(*tok, i).await)
});
Self { range: tt.range.clone(), tok }
}
pub async fn to_api(&self, do_extra: &mut impl RefDoExtra<X>) -> api::TokenTree {
let token = match_mapping!(&self.tok, Token => api::Token {
Atom(a.to_api().await),
impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
pub async fn from_api(
tt: &api::TokenTree,
hctx: &mut H::FromApiCtx<'_>,
xctx: &mut X::FromApiCtx<'_>,
src: &Sym,
i: &Interner,
) -> Self {
let pos = SrcRange::new(tt.range.clone(), src);
let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> {
BR,
NS,
Bottom(e.to_api()),
NS(n => Tok::from_api(*n, i).await,
b => Box::new(Self::from_api(b, hctx, xctx, src, i).boxed_local().await)),
Bottom(e => OrcErrv::from_api(e, i).await),
LambdaHead(arg => ttv_from_api(arg, hctx, xctx, src, i).await),
Name(n => Tok::from_api(*n, i).await),
S(*par, b => ttv_from_api(b, hctx, xctx, src, i).await),
Comment(c.clone()),
LambdaHead(arg => ttv_to_api(arg, do_extra).boxed_local().await),
Name(n.to_api()),
Slot(tt.ticket()),
S(*p, b => ttv_to_api(b, do_extra).boxed_local().await),
Ph(ph.to_api()),
Macro(*prio),
Reference(sym.to_api()),
} {
Token::X(x) => return do_extra(x, self.range.clone()).await
NewExpr(expr => X::from_api(expr, xctx, pos.clone(), i).await),
Handle(tk => H::from_api(tk, hctx, pos.clone(), i).await)
});
api::TokenTree { range: self.range.clone(), token }
Self { sr: pos, tok }
}
pub async fn into_api(
self,
do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
hctx: &mut H::ToApiCtx<'_>,
xctx: &mut X::ToApiCtx<'_>,
) -> api::TokenTree {
let token = match self.tok {
Token::Atom(a) => api::Token::Atom(a.to_api().await),
Token::Reference(sym) => api::Token::Reference(sym.to_api()),
Token::BR => api::Token::BR,
Token::NS => api::Token::NS,
Token::Bottom(e) => api::Token::Bottom(e.to_api()),
Token::Comment(c) => api::Token::Comment(c.clone()),
Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra).await),
Token::Name(n) => api::Token::Name(n.to_api()),
Token::Slot(tt) => api::Token::Slot(tt.ticket()),
Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra).await),
Token::Ph(Ph { kind, name }) =>
api::Token::Ph(api::Placeholder { name: name.to_api(), kind }),
Token::X(x) => return do_extra(x, self.range.clone()),
Token::Macro(prio) => api::Token::Macro(prio),
};
api::TokenTree { range: self.range.clone(), token }
let token = match_mapping!(self.tok, Token => api::Token {
BR,
NS(n.to_api(), b => Box::new(b.into_api(hctx, xctx).boxed_local().await)),
Bottom(e.to_api()),
Comment(c.clone()),
LambdaHead(arg => ttv_into_api(arg, hctx, xctx).boxed_local().await),
Name(nn.to_api()),
S(p, b => ttv_into_api(b, hctx, xctx).boxed_local().await),
Handle(hand.into_api(hctx).await),
NewExpr(expr.into_api(xctx).await),
});
api::TokenTree { range: self.sr.range.clone(), token }
}
pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
pub fn as_name(&self) -> Option<Tok<String>> {
if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
}
pub fn as_s<'a>(&'a self, par: Paren, i: &'a Interner) -> Option<Snippet<'a, 'b, A, X>> {
self.tok.as_s(par).map(|slc| Snippet::new(self, slc, i))
pub fn as_s(&self, par: Paren) -> Option<Snippet<'_, H, X>> {
self.tok.as_s(par).map(|slc| Snippet::new(self, slc))
}
pub fn as_lambda(&self) -> Option<Snippet<'_, H, X>> {
match &self.tok {
Token::LambdaHead(arg) => Some(Snippet::new(self, arg)),
_ => None,
}
}
pub fn is_fluff(&self) -> bool { matches!(self.tok, Token::Comment(_) | Token::BR) }
pub fn lambda(arg: Vec<Self>, mut body: Vec<Self>) -> Self {
let arg_range = ttv_range(&arg);
let s_range = arg_range.start..body.last().expect("Lambda with empty body!").range.end;
let arg_range = ttv_range(&arg).expect("Lambda with empty arg!");
let mut s_range = arg_range.clone();
s_range.range.end = body.last().expect("Lambda with empty body!").sr.range.end;
body.insert(0, Token::LambdaHead(arg).at(arg_range));
Token::S(Paren::Round, body).at(s_range)
}
pub fn sr(&self) -> SrcRange { self.sr.clone() }
}
impl<A: AtomRepr, X: ExtraTok> Format for TokTree<'_, A, X> {
impl<H: ExprRepr, X: ExtraTok> Format for TokTree<H, X> {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
self.tok.print(c).await
}
}
pub async fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
ctx: &mut A::Ctx,
hctx: &mut H::FromApiCtx<'_>,
xctx: &mut X::FromApiCtx<'_>,
src: &Sym,
i: &Interner,
) -> Vec<TokTree<'static, A, X>> {
) -> Vec<TokTree<H, X>> {
stream! {
for tok in tokv {
yield TokTree::<A, X>::from_api(tok.borrow(), ctx, i).boxed_local().await
yield TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await
}
}
.collect()
.await
}
pub async fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>(
tokv: impl IntoIterator<Item: Borrow<TokTree<'a, A, X>>>,
do_extra: &mut impl RefDoExtra<X>,
) -> Vec<api::TokenTree> {
let mut output = Vec::new();
for tok in tokv {
output.push(Borrow::<TokTree<A, X>>::borrow(&tok).to_api(do_extra).await)
}
output
}
pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
tokv: impl IntoIterator<Item = TokTree<'a, A, X>>,
do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
pub async fn ttv_into_api<H: ExprRepr, X: ExtraTok>(
tokv: impl IntoIterator<Item = TokTree<H, X>>,
hctx: &mut H::ToApiCtx<'_>,
xctx: &mut X::ToApiCtx<'_>,
) -> Vec<api::TokenTree> {
stream! {
for tok in tokv {
yield tok.into_api(do_extra).await
yield tok.into_api(hctx, xctx).await
}
}
.collect()
.await
}
pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
items: impl IntoIterator<Item = TokTree<'a, A, X>>,
) -> TokTree<'a, A, X> {
pub fn wrap_tokv<H: ExprRepr, X: ExtraTok>(
items: impl IntoIterator<Item = TokTree<H, X>>,
) -> TokTree<H, X> {
let items_v = items.into_iter().collect_vec();
match items_v.len() {
0 => panic!("A tokv with no elements is illegal"),
1 => items_v.into_iter().next().unwrap(),
_ => {
let range = items_v.first().unwrap().range.start..items_v.last().unwrap().range.end;
Token::S(api::Paren::Round, items_v).at(range)
let sr = ttv_range(&items_v).expect("empty handled above");
Token::S(api::Paren::Round, items_v).at(sr)
},
}
}
@@ -212,70 +223,54 @@ pub use api::Paren;
/// Lexer output variant
#[derive(Clone, Debug)]
pub enum Token<'a, A: AtomRepr, X: ExtraTok> {
pub enum Token<H: ExprRepr, X: ExtraTok> {
/// Information about the code addressed to the human reader or dev tooling
/// It has no effect on the behaviour of the program unless it's explicitly
/// read via reflection
Comment(Arc<String>),
Comment(Rc<String>),
/// The part of a lambda between `\` and `.` enclosing the argument. The body
/// stretches to the end of the enclosing parens or the end of the const line
LambdaHead(Vec<TokTree<'a, A, X>>),
LambdaHead(Vec<TokTree<H, X>>),
/// A binding, operator, or a segment of a namespaced::name
Name(Tok<String>),
/// The namespace separator ::
NS,
/// A namespace prefix, like `my_ns::` followed by a token
NS(Tok<String>, Box<TokTree<H, X>>),
/// A line break
BR,
/// `()`, `[]`, or `{}`
S(Paren, Vec<TokTree<'a, A, X>>),
/// A fully formed reference to external code emitted by a lexer plugin
Reference(Sym),
/// A value emitted by a lexer plugin
Atom(A),
S(Paren, Vec<TokTree<H, X>>),
/// A newly instantiated expression
NewExpr(X),
/// An existing expr from a nested lexer
Handle(H),
/// A grammar error emitted by a lexer plugin if it was possible to continue
/// reading. Parsers should treat it as an atom unless it prevents parsing,
/// in which case both this and a relevant error should be returned.
Bottom(OrcErrv),
/// An instruction from a plugin for the lexer to embed a subexpression
/// without retransmitting it. It should not appear anywhere outside lexer
/// plugin responses.
Slot(TokHandle<'a>),
/// Additional domain-specific token types
X(X),
/// A placeholder for metaprogramming, either $name, ..$name, ..$name:N,
/// ...$name, or ...$name:N
Ph(Ph),
/// `macro` or `macro(`X`)` where X is any valid floating point number
/// expression. `macro` is not a valid name in Orchid for this reason.
Macro(Option<NotNan<f64>>),
}
impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
pub fn at(self, range: Range<u32>) -> TokTree<'a, A, X> { TokTree { range, tok: self } }
impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) }
pub fn as_s(&self, par: Paren) -> Option<&[TokTree<'a, A, X>]> {
pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
match self {
Self::S(p, b) if *p == par => Some(b),
_ => None,
}
}
}
impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
impl<H: ExprRepr, X: ExtraTok> Format for Token<H, X> {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
match self {
Self::Atom(a) => a.print(c).await,
Self::BR => "\n".to_string().into(),
Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()).into(),
Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),
Self::Comment(c) => format!("--[{c}]--").into(),
Self::LambdaHead(arg) =>
FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0b}."))), [
ttv_fmt(arg, c).await,
]),
Self::NS => "::".to_string().into(),
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0b}.")))
.units([ttv_fmt(arg, c).await]),
Self::NS(n, b) => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}::{1l}")))
.units([n.to_string().into(), b.print(c).boxed_local().await]),
Self::Name(n) => format!("{n}").into(),
Self::Reference(sym) => format!("{sym}").into(),
Self::Slot(th) => format!("{th}").into(),
Self::Ph(ph) => format!("{ph}").into(),
Self::S(p, b) => FmtUnit::new(
match *p {
Paren::Round => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("({0b})"))),
@@ -284,67 +279,22 @@ impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
},
[ttv_fmt(b, c).await],
),
Self::X(x) => x.print(c).await,
Self::Macro(None) => "macro".to_string().into(),
Self::Macro(Some(prio)) => format!("macro({prio})").into(),
Self::Handle(h) => h.print(c).await,
Self::NewExpr(ex) => ex.print(c).await,
}
}
}
pub fn ttv_range(ttv: &[TokTree<'_, impl AtomRepr, impl ExtraTok>]) -> Range<u32> {
assert!(!ttv.is_empty(), "Empty slice has no range");
ttv.first().unwrap().range.start..ttv.last().unwrap().range.end
pub fn ttv_range<'a>(ttv: &[TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>]) -> Option<SrcRange> {
let range = ttv.first()?.sr.range.start..ttv.last().unwrap().sr.range.end;
Some(SrcRange { path: ttv.first().unwrap().sr.path(), range })
}
pub async fn ttv_fmt<'a: 'b, 'b>(
ttv: impl IntoIterator<Item = &'b TokTree<'a, impl AtomRepr + 'b, impl ExtraTok + 'b>>,
ttv: impl IntoIterator<Item = &'b TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>>,
c: &(impl FmtCtx + ?Sized),
) -> FmtUnit {
FmtUnit::sequence(" ", None, join_all(ttv.into_iter().map(|t| t.print(c))).await)
}
pub fn indent(s: &str) -> String { s.replace("\n", "\n ") }
#[derive(Clone, Debug)]
pub struct Ph {
pub name: Tok<String>,
pub kind: PhKind,
}
impl Ph {
pub async fn from_api(api: &api::Placeholder, i: &Interner) -> Self {
Self { name: Tok::from_api(api.name, i).await, kind: api.kind }
}
pub fn to_api(&self) -> api::Placeholder {
api::Placeholder { name: self.name.to_api(), kind: self.kind }
}
}
impl Display for Ph {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let n = &self.name;
match self.kind {
PhKind::Scalar => write!(f, "${n}"),
PhKind::Vector { priority: 0, at_least_one: true } => write!(f, "...${}", self.name),
PhKind::Vector { priority: p, at_least_one: true } => write!(f, "...${}:{}", self.name, p),
PhKind::Vector { priority: 0, at_least_one: false } => write!(f, "..${}", self.name),
PhKind::Vector { priority: p, at_least_one: false } => write!(f, "..${}:{}", self.name, p),
}
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_covariance() {
fn _f<'a>(x: Token<'static, Never, Never>) -> Token<'a, Never, Never> { x }
}
#[test]
fn fail_covariance() {
// this fails to compile
// fn _f<'a, 'b>(x: &'a mut &'static ()) -> &'a mut &'b () { x }
// this passes because it's covariant
fn _f<'a, 'b>(x: &'a fn() -> &'static ()) -> &'a fn() -> &'b () { x }
}
}

11
orchid-extension/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "orchid-extension"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,10 +9,12 @@ edition = "2021"
ahash = "0.8.11"
async-once-cell = "0.5.4"
async-std = "1.13.0"
async-stream = "0.3.6"
derive_destructure = "1.0.0"
dyn-clone = "1.0.17"
futures = "0.3.31"
hashbrown = "0.15.2"
include_dir = { version = "0.7.4", optional = true }
itertools = "0.14.0"
konst = "0.3.16"
lazy_static = "1.5.0"
@@ -23,8 +25,9 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
orchid-base = { version = "0.1.0", path = "../orchid-base" }
ordered-float = "4.6.0"
paste = "1.0.15"
some_executor = "0.4.0"
ordered-float = "5.0.0"
pastey = "0.1.0"
some_executor = "0.5.1"
substack = "1.1.1"
tokio = { version = "1.46.1", optional = true }
trait-set = "0.3.0"

89
orchid-extension/src/atom.rs
View File

@@ -1,7 +1,6 @@
use std::any::{Any, TypeId, type_name};
use std::fmt;
use std::future::Future;
use std::marker::PhantomData;
use std::num::NonZeroU32;
use std::ops::Deref;
use std::pin::Pin;
@@ -22,7 +21,6 @@ use orchid_base::interner::Interner;
use orchid_base::location::Pos;
use orchid_base::name::Sym;
use orchid_base::reqnot::Requester;
use orchid_base::tree::AtomRepr;
use trait_set::trait_set;
use crate::api;
@@ -42,7 +40,10 @@ pub trait AtomicVariant {}
pub trait Atomic: 'static + Sized {
type Variant: AtomicVariant;
type Data: Clone + Coding + Sized + 'static;
fn reg_reqs() -> MethodSetBuilder<Self>;
/// Register handlers for IPC calls. If this atom implements [Supports], you
/// should register your implementations here. If this atom doesn't
/// participate in IPC at all, the default implementation is fine
fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
}
impl<A: Atomic> AtomCard for A {
type Data = <Self as Atomic>::Data;
@@ -84,57 +85,43 @@ pub fn get_info<A: AtomCard>(
}
#[derive(Clone)]
pub struct ForeignAtom<'a> {
pub(crate) expr: Option<Rc<ExprHandle>>,
pub(crate) _life: PhantomData<&'a ()>,
pub(crate) ctx: SysCtx,
pub struct ForeignAtom {
pub(crate) expr: Rc<ExprHandle>,
pub(crate) atom: api::Atom,
pub(crate) pos: Pos,
}
impl ForeignAtom<'_> {
pub fn ex_opt(self) -> Option<Expr> {
let (handle, pos) = (self.expr.as_ref()?.clone(), self.pos.clone());
let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { _life: PhantomData, ..self }) };
Some(Expr::new(handle, data))
}
impl ForeignAtom {
pub fn pos(&self) -> Pos { self.pos.clone() }
pub fn ctx(&self) -> SysCtx { self.ctx.clone() }
}
impl ForeignAtom<'static> {
pub fn ex(self) -> Expr { self.ex_opt().unwrap() }
pub fn ctx(&self) -> SysCtx { self.expr.ctx.clone() }
pub fn ex(self) -> Expr {
let (handle, pos) = (self.expr.clone(), self.pos.clone());
let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { ..self }) };
Expr::new(handle, data)
}
pub(crate) fn new(handle: Rc<ExprHandle>, atom: api::Atom, pos: Pos) -> Self {
ForeignAtom { _life: PhantomData, atom, ctx: handle.ctx.clone(), expr: Some(handle), pos }
ForeignAtom { atom, expr: handle, pos }
}
pub async fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
let rep = (self.ctx.reqnot.request(api::Fwd(
let rep = (self.ctx().reqnot().request(api::Fwd(
self.atom.clone(),
Sym::parse(M::NAME, &self.ctx.i).await.unwrap().tok().to_api(),
Sym::parse(M::NAME, self.ctx().i()).await.unwrap().tok().to_api(),
enc_vec(&m).await,
)))
.await?;
Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
}
}
impl fmt::Display for ForeignAtom<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}::{:?}", if self.expr.is_some() { "Clause" } else { "Tok" }, self.atom)
}
impl fmt::Display for ForeignAtom {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Atom::{:?}", self.atom) }
}
impl fmt::Debug for ForeignAtom<'_> {
impl fmt::Debug for ForeignAtom {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ForeignAtom({self})") }
}
impl Format for ForeignAtom<'_> {
impl Format for ForeignAtom {
async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
FmtUnit::from_api(&self.ctx.reqnot.request(api::ExtAtomPrint(self.atom.clone())).await)
FmtUnit::from_api(&self.ctx().reqnot().request(api::ExtAtomPrint(self.atom.clone())).await)
}
}
impl AtomRepr for ForeignAtom<'_> {
type Ctx = SysCtx;
async fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self {
Self { atom: atom.clone(), _life: PhantomData, ctx: ctx.clone(), expr: None, pos }
}
async fn to_api(&self) -> orchid_api::Atom { self.atom.clone() }
}
pub struct NotTypAtom {
pub pos: Pos,
@@ -145,7 +132,7 @@ pub struct NotTypAtom {
impl NotTypAtom {
pub async fn mk_err(&self) -> OrcErr {
mk_err(
self.ctx.i.i("Not the expected type").await,
self.ctx.i().i("Not the expected type").await,
format!("This expression is not a {}", self.typ.name()),
[self.pos.clone().into()],
)
@@ -192,7 +179,7 @@ impl<A: AtomCard> MethodSetBuilder<A> {
handlers: stream::from_iter(self.handlers.iter())
.then(|(k, v)| {
clone!(ctx; async move {
(Sym::parse(k, &ctx.i).await.unwrap(), v.clone())
(Sym::parse(k, ctx.i()).await.unwrap(), v.clone())
})
})
.collect()
@@ -228,11 +215,11 @@ impl<A: AtomCard> Default for MethodSetBuilder<A> {
}
#[derive(Clone)]
pub struct TypAtom<'a, A: AtomicFeatures> {
pub data: ForeignAtom<'a>,
pub struct TypAtom<A: AtomicFeatures> {
pub data: ForeignAtom,
pub value: A::Data,
}
impl<A: AtomicFeatures> TypAtom<'static, A> {
impl<A: AtomicFeatures> TypAtom<A> {
pub async fn downcast(expr: Rc<ExprHandle>) -> Result<Self, NotTypAtom> {
match Expr::from_handle(expr).atom().await {
Err(expr) => Err(NotTypAtom {
@@ -245,21 +232,19 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
Ok(tatom) => Ok(tatom),
Err(fa) => Err(NotTypAtom {
pos: fa.pos.clone(),
ctx: fa.ctx.clone(),
ctx: fa.ctx().clone(),
expr: fa.ex(),
typ: Box::new(A::info()),
}),
},
}
}
}
impl<A: AtomicFeatures> TypAtom<'_, A> {
pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
where A: Supports<M> {
M::Response::decode(Pin::new(
&mut &(self.data.ctx.reqnot.request(api::Fwd(
&mut &(self.data.ctx().reqnot().request(api::Fwd(
self.data.atom.clone(),
Sym::parse(M::NAME, &self.data.ctx.i).await.unwrap().tok().to_api(),
Sym::parse(M::NAME, self.data.ctx().i()).await.unwrap().tok().to_api(),
enc_vec(&req).await,
)))
.await
@@ -268,22 +253,22 @@ impl<A: AtomicFeatures> TypAtom<'_, A> {
.await
}
}
impl<A: AtomicFeatures> Deref for TypAtom<'_, A> {
impl<A: AtomicFeatures> Deref for TypAtom<A> {
type Target = A::Data;
fn deref(&self) -> &Self::Target { &self.value }
}
pub struct AtomCtx<'a>(pub &'a [u8], pub Option<api::AtomId>, pub SysCtx);
impl FmtCtx for AtomCtx<'_> {
fn i(&self) -> &Interner { &self.2.i }
fn i(&self) -> &Interner { self.2.i() }
}
pub trait AtomDynfo: 'static {
fn tid(&self) -> TypeId;
fn name(&self) -> &'static str;
fn decode<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>>;
fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
fn print<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit>;
fn handle_req<'a, 'b: 'a, 'c: 'a>(
&'a self,
@@ -297,12 +282,12 @@ pub trait AtomDynfo: 'static {
&'a self,
ctx: AtomCtx<'a>,
write: Pin<&'b mut dyn Write>,
) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>>;
) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
fn deserialize<'a>(
&'a self,
ctx: SysCtx,
data: &'a [u8],
refs: &'a [api::ExprTicket],
refs: &'a [Expr],
) -> LocalBoxFuture<'a, api::Atom>;
fn drop<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, ()>;
}
@@ -312,9 +297,7 @@ trait_set! {
}
pub struct AtomFactory(Box<dyn AtomFactoryFn>);
impl AtomFactory {
pub fn new<F: Future<Output = api::Atom> + 'static>(
f: impl FnOnce(SysCtx) -> F + Clone + 'static,
) -> Self {
pub fn new(f: impl AsyncFnOnce(SysCtx) -> api::Atom + Clone + 'static) -> Self {
Self(Box::new(|ctx| f(ctx).boxed_local()))
}
pub async fn build(self, ctx: SysCtx) -> api::Atom { (self.0)(ctx).await }

117
orchid-extension/src/atom_owned.rs
View File

@@ -4,7 +4,6 @@ use std::future::Future;
use std::num::NonZero;
use std::ops::Deref;
use std::pin::Pin;
use std::rc::Rc;
use std::sync::atomic::AtomicU64;
use async_once_cell::OnceCell;
@@ -18,7 +17,7 @@ use never::Never;
use orchid_api::AtomId;
use orchid_api_traits::{Decode, Encode, enc_vec};
use orchid_base::error::OrcRes;
use orchid_base::format::FmtUnit;
use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit};
use orchid_base::name::Sym;
use crate::api;
@@ -26,22 +25,25 @@ use crate::atom::{
AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
MethodSetBuilder, err_not_callable, err_not_command, get_info,
};
use crate::expr::{Expr, ExprHandle};
use crate::expr::Expr;
use crate::gen_expr::{GExpr, bot};
use crate::system::SysCtx;
use crate::system::{SysCtx, SysCtxEntry};
use crate::system_ctor::CtedObj;
pub struct OwnedVariant;
impl AtomicVariant for OwnedVariant {}
impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVariant> for A {
fn _factory(self) -> AtomFactory {
AtomFactory::new(move |ctx| async move {
let serial = ctx.obj_store.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
AtomFactory::new(async move |ctx| {
let serial =
ctx.get_or_default::<ObjStore>().next_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let atom_id = api::AtomId(NonZero::new(serial + 1).unwrap());
let (typ_id, _) = get_info::<A>(ctx.cted.inst().card());
let (typ_id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
let mut data = enc_vec(&typ_id).await;
self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
ctx.obj_store.1.read().await.insert(atom_id, Box::new(self));
api::Atom { drop: Some(atom_id), data, owner: ctx.id }
ctx.get_or_default::<ObjStore>().objects.read().await.insert(atom_id, Box::new(self));
eprintln!("Created atom {:?} of type {}", atom_id, type_name::<A>());
api::Atom { drop: Some(atom_id), data, owner: ctx.sys_id() }
})
}
fn _info() -> Self::_Info { OwnedAtomDynfo { msbuild: A::reg_reqs(), ms: OnceCell::new() } }
@@ -55,8 +57,9 @@ pub(crate) struct AtomReadGuard<'a> {
}
impl<'a> AtomReadGuard<'a> {
async fn new(id: api::AtomId, ctx: &'a SysCtx) -> Self {
let guard = ctx.obj_store.1.read().await;
assert!(guard.get(&id).is_some(), "Received invalid atom ID: {}", id.0);
let guard = ctx.get_or_default::<ObjStore>().objects.read().await;
let valid = guard.iter().map(|i| i.0).collect_vec();
assert!(guard.get(&id).is_some(), "Received invalid atom ID: {id:?} not in {valid:?}");
Self { id, guard }
}
}
@@ -66,7 +69,7 @@ impl Deref for AtomReadGuard<'_> {
}
pub(crate) async fn take_atom(id: api::AtomId, ctx: &SysCtx) -> Box<dyn DynOwnedAtom> {
let mut g = ctx.obj_store.1.write().await;
let mut g = ctx.get_or_default::<ObjStore>().objects.write().await;
g.remove(&id).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0))
}
@@ -78,25 +81,24 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
fn tid(&self) -> TypeId { TypeId::of::<T>() }
fn name(&self) -> &'static str { type_name::<T>() }
fn decode<'a>(&'a self, AtomCtx(data, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
async {
Box::pin(async {
Box::new(<T as AtomCard>::Data::decode(Pin::new(&mut &data[..])).await) as Box<dyn Any>
}
.boxed_local()
})
}
fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
async move { take_atom(id.unwrap(), &ctx).await.dyn_call(ctx.clone(), arg).await }.boxed_local()
fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_call(arg).await })
}
fn call_ref<'a>(
&'a self,
AtomCtx(_, id, ctx): AtomCtx<'a>,
arg: api::ExprTicket,
arg: Expr,
) -> LocalBoxFuture<'a, GExpr> {
async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_call_ref(ctx.clone(), arg).await }
.boxed_local()
Box::pin(async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_call_ref(arg).await })
}
fn print(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> LocalBoxFuture<'_, FmtUnit> {
async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_print(ctx.clone()).await }
.boxed_local()
Box::pin(
async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_print(ctx.clone()).await },
)
}
fn handle_req<'a, 'b: 'a, 'c: 'a>(
&'a self,
@@ -105,48 +107,43 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
req: Pin<&'b mut dyn Read>,
rep: Pin<&'c mut dyn Write>,
) -> LocalBoxFuture<'a, bool> {
async move {
Box::pin(async move {
let a = AtomReadGuard::new(id.unwrap(), &ctx).await;
let ms = self.ms.get_or_init(self.msbuild.pack(ctx.clone())).await;
ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx.clone(), key, req, rep).await
}
.boxed_local()
})
}
fn command<'a>(
&'a self,
AtomCtx(_, id, ctx): AtomCtx<'a>,
) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>> {
async move { take_atom(id.unwrap(), &ctx).await.dyn_command(ctx.clone()).await }.boxed_local()
Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_command(ctx.clone()).await })
}
fn drop(&self, AtomCtx(_, id, ctx): AtomCtx) -> LocalBoxFuture<'_, ()> {
async move { take_atom(id.unwrap(), &ctx).await.dyn_free(ctx.clone()).await }.boxed_local()
Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_free(ctx.clone()).await })
}
fn serialize<'a, 'b: 'a>(
&'a self,
AtomCtx(_, id, ctx): AtomCtx<'a>,
mut write: Pin<&'b mut dyn Write>,
) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
async move {
) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
Box::pin(async move {
let id = id.unwrap();
id.encode(write.as_mut()).await;
let refs = AtomReadGuard::new(id, &ctx).await.dyn_serialize(ctx.clone(), write).await;
refs.map(|v| v.into_iter().map(|t| t.handle().tk).collect_vec())
}
.boxed_local()
AtomReadGuard::new(id, &ctx).await.dyn_serialize(ctx.clone(), write).await
})
}
fn deserialize<'a>(
&'a self,
ctx: SysCtx,
data: &'a [u8],
refs: &'a [api::ExprTicket],
refs: &'a [Expr],
) -> LocalBoxFuture<'a, api::Atom> {
async move {
let refs =
refs.iter().map(|tk| Expr::from_handle(Rc::new(ExprHandle::from_args(ctx.clone(), *tk))));
let obj = T::deserialize(DeserCtxImpl(data, &ctx), T::Refs::from_iter(refs)).await;
Box::pin(async move {
let refs = T::Refs::from_iter(refs.iter().cloned());
let obj = T::deserialize(DeserCtxImpl(data, &ctx), refs).await;
obj._factory().build(ctx).await
}
.boxed_local()
})
}
}
@@ -218,12 +215,12 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
type Refs: RefSet;
fn val(&self) -> impl Future<Output = Cow<'_, Self::Data>>;
#[allow(unused_variables)]
fn call_ref(&self, arg: ExprHandle) -> impl Future<Output = GExpr> {
async move { bot([err_not_callable(&arg.ctx.i).await]) }
fn call_ref(&self, arg: Expr) -> impl Future<Output = GExpr> {
async move { bot([err_not_callable(arg.ctx().i()).await]) }
}
fn call(self, arg: ExprHandle) -> impl Future<Output = GExpr> {
fn call(self, arg: Expr) -> impl Future<Output = GExpr> {
async {
let ctx = arg.get_ctx();
let ctx = arg.ctx();
let gcl = self.call_ref(arg).await;
self.free(ctx).await;
gcl
@@ -231,12 +228,12 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
}
#[allow(unused_variables)]
fn command(self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
async move { Err(err_not_command(&ctx.i).await.into()) }
async move { Err(err_not_command(ctx.i()).await.into()) }
}
#[allow(unused_variables)]
fn free(self, ctx: SysCtx) -> impl Future<Output = ()> { async {} }
#[allow(unused_variables)]
fn print(&self, ctx: SysCtx) -> impl Future<Output = FmtUnit> {
fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> {
async { format!("OwnedAtom({})", type_name::<Self>()).into() }
}
#[allow(unused_variables)]
@@ -266,9 +263,8 @@ pub trait DynOwnedAtom: 'static {
fn atom_tid(&self) -> TypeId;
fn as_any_ref(&self) -> &dyn Any;
fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()>;
fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr>;
fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket)
-> LocalBoxFuture<'static, GExpr>;
fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr>;
fn dyn_call(self: Box<Self>, arg: Expr) -> LocalBoxFuture<'static, GExpr>;
fn dyn_command(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, OrcRes<Option<GExpr>>>;
fn dyn_free(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, ()>;
fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit>;
@@ -284,15 +280,11 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()> {
async { self.val().await.as_ref().encode(buffer).await }.boxed_local()
}
fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
self.call_ref(ExprHandle::from_args(ctx, arg)).boxed_local()
fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
self.call_ref(arg).boxed_local()
}
fn dyn_call(
self: Box<Self>,
ctx: SysCtx,
arg: api::ExprTicket,
) -> LocalBoxFuture<'static, GExpr> {
self.call(ExprHandle::from_args(ctx, arg)).boxed_local()
fn dyn_call(self: Box<Self>, arg: Expr) -> LocalBoxFuture<'static, GExpr> {
self.call(arg).boxed_local()
}
fn dyn_command(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, OrcRes<Option<GExpr>>> {
self.command(ctx).boxed_local()
@@ -300,7 +292,9 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
fn dyn_free(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, ()> {
self.free(ctx).boxed_local()
}
fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit> { self.print(ctx).boxed_local() }
fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit> {
async move { self.print(&FmtCtxImpl { i: ctx.i() }).await }.boxed_local()
}
fn dyn_serialize<'a>(
&'a self,
ctx: SysCtx,
@@ -313,4 +307,9 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
}
}
pub type ObjStore = Rc<(AtomicU64, RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>)>;
#[derive(Default)]
struct ObjStore {
next_id: AtomicU64,
objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
}
impl SysCtxEntry for ObjStore {}

52
orchid-extension/src/atom_thin.rs
View File

@@ -16,19 +16,20 @@ use crate::atom::{
AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
MethodSetBuilder, err_not_callable, err_not_command, get_info,
};
use crate::expr::ExprHandle;
use crate::expr::Expr;
use crate::gen_expr::{GExpr, bot};
use crate::system::SysCtx;
use crate::system_ctor::CtedObj;
pub struct ThinVariant;
impl AtomicVariant for ThinVariant {}
impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant> for A {
fn _factory(self) -> AtomFactory {
AtomFactory::new(move |ctx| async move {
let (id, _) = get_info::<A>(ctx.cted.inst().card());
AtomFactory::new(async move |ctx| {
let (id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
let mut buf = enc_vec(&id).await;
self.encode(Pin::new(&mut buf)).await;
api::Atom { drop: None, data: buf, owner: ctx.id }
api::Atom { drop: None, data: buf, owner: ctx.sys_id() }
})
}
fn _info() -> Self::_Info { ThinAtomDynfo { msbuild: Self::reg_reqs(), ms: OnceCell::new() } }
@@ -41,30 +42,18 @@ pub struct ThinAtomDynfo<T: ThinAtom> {
}
impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
fn print<'a>(&self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit> {
async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await }.boxed_local()
Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await })
}
fn tid(&self) -> TypeId { TypeId::of::<T>() }
fn name(&self) -> &'static str { type_name::<T>() }
fn decode<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> }.boxed_local()
Box::pin(async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> })
}
fn call<'a>(
&'a self,
AtomCtx(buf, _, ctx): AtomCtx<'a>,
arg: api::ExprTicket,
) -> LocalBoxFuture<'a, GExpr> {
Box::pin(async move {
T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
})
fn call<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
}
fn call_ref<'a>(
&'a self,
AtomCtx(buf, _, ctx): AtomCtx<'a>,
arg: api::ExprTicket,
) -> LocalBoxFuture<'a, GExpr> {
Box::pin(async move {
T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
})
fn call_ref<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
}
fn handle_req<'a, 'm1: 'a, 'm2: 'a>(
&'a self,
@@ -88,7 +77,7 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
&'a self,
ctx: AtomCtx<'a>,
write: Pin<&'b mut dyn Write>,
) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
Box::pin(async {
T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await;
Some(Vec::new())
@@ -98,17 +87,16 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
&'a self,
ctx: SysCtx,
data: &'a [u8],
refs: &'a [api::ExprTicket],
refs: &'a [Expr],
) -> LocalBoxFuture<'a, api::Atom> {
assert!(refs.is_empty(), "Refs found when deserializing thin atom");
async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await }.boxed_local()
Box::pin(async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await })
}
fn drop<'a>(&'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> {
async move {
Box::pin(async move {
let string_self = T::decode(Pin::new(&mut &buf[..])).await.print(ctx.clone()).await;
writeln!(ctx.logger, "Received drop signal for non-drop atom {string_self:?}");
}
.boxed_local()
writeln!(ctx.logger(), "Received drop signal for non-drop atom {string_self:?}");
})
}
}
@@ -116,12 +104,12 @@ pub trait ThinAtom:
AtomCard<Data = Self> + Atomic<Variant = ThinVariant> + Coding + Send + Sync + 'static
{
#[allow(unused_variables)]
fn call(&self, arg: ExprHandle) -> impl Future<Output = GExpr> {
async move { bot([err_not_callable(&arg.ctx.i).await]) }
fn call(&self, arg: Expr) -> impl Future<Output = GExpr> {
async move { bot([err_not_callable(arg.ctx().i()).await]) }
}
#[allow(unused_variables)]
fn command(&self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
async move { Err(err_not_command(&ctx.i).await.into()) }
async move { Err(err_not_command(ctx.i()).await.into()) }
}
#[allow(unused_variables)]
fn print(&self, ctx: SysCtx) -> impl Future<Output = FmtUnit> {

8
orchid-extension/src/conv.rs
View File

@@ -31,13 +31,13 @@ async fn err_type(pos: Pos, i: &Interner) -> OrcErr {
mk_err(i.i("Type error").await, "The atom is a different type than expected", [pos.into()])
}
impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {
impl<A: AtomicFeatures> TryFromExpr for TypAtom<A> {
async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
match expr.atom().await {
Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &ex.ctx().i).await.into()),
Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), ex.ctx().i()).await.into()),
Ok(f) => match downcast_atom::<A>(f).await {
Ok(a) => Ok(a),
Err(f) => Err(err_type(f.pos(), &f.ctx().i).await.into()),
Err(f) => Err(err_type(f.pos(), f.ctx().i()).await.into()),
},
}
}
@@ -51,7 +51,7 @@ impl ToExpr for GExpr {
fn to_expr(self) -> GExpr { self }
}
impl ToExpr for Expr {
fn to_expr(self) -> GExpr { self.gen() }
fn to_expr(self) -> GExpr { self.slot() }
}
impl<T: ToExpr> ToExpr for OrcRes<T> {

421
orchid-extension/src/entrypoint.rs
View File

@@ -1,45 +1,40 @@
use std::cell::RefCell;
use std::future::Future;
use std::io::Write;
use std::mem;
use std::num::NonZero;
use std::pin::Pin;
use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use async_std::channel::{Receiver, Sender};
use async_std::channel::{self, Receiver, Sender};
use async_std::stream;
use async_std::sync::Mutex;
use futures::future::{LocalBoxFuture, join_all};
use futures::{FutureExt, StreamExt};
use futures::{FutureExt, StreamExt, stream_select};
use hashbrown::HashMap;
use itertools::Itertools;
use orchid_api::ApplyMacro;
use orchid_api_traits::{Decode, Encode, enc_vec};
use orchid_base::builtin::{ExtPort, Spawner};
use orchid_api::{ExtMsgSet, IntReq};
use orchid_api_traits::{Decode, UnderRoot, enc_vec};
use orchid_base::builtin::{ExtInit, ExtPort, Spawner};
use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
use orchid_base::clone;
use orchid_base::interner::{Interner, Tok};
use orchid_base::logging::Logger;
use orchid_base::macros::{mtreev_from_api, mtreev_to_api};
use orchid_base::name::Sym;
use orchid_base::parse::{Comment, Snippet};
use orchid_base::reqnot::{ReqNot, RequestHandle, Requester};
use orchid_base::tree::{ttv_from_api, ttv_to_api};
use orchid_base::tree::{TokenVariant, ttv_from_api};
use substack::Substack;
use trait_set::trait_set;
use crate::api;
use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId};
use crate::atom_owned::{ObjStore, take_atom};
use crate::fs::VirtFS;
use crate::atom_owned::take_atom;
use crate::expr::{Expr, ExprHandle};
use crate::lexer::{LexContext, err_cascade, err_not_applicable};
use crate::macros::{Rule, RuleCtx};
use crate::msg::{recv_parent_msg, send_parent_msg};
use crate::parser::{ParsCtx, get_const, linev_into_api};
use crate::system::{SysCtx, atom_by_idx};
use crate::system_ctor::{CtedObj, DynSystemCtor};
use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TIACtxImpl, do_extra};
use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TreeIntoApiCtxImpl};
pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
@@ -52,7 +47,6 @@ impl ExtensionData {
pub fn new(name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
Self { name, systems }
}
// pub fn main(self) { extension_main(self) }
}
pub enum MemberRecord {
@@ -61,123 +55,106 @@ pub enum MemberRecord {
}
pub struct SystemRecord {
cted: CtedObj,
vfses: HashMap<api::VfsId, &'static dyn VirtFS>,
declfs: api::EagerVfs,
lazy_members: HashMap<api::TreeId, MemberRecord>,
rules: HashMap<api::MacroId, Rc<Rule>>,
ctx: SysCtx,
}
trait_set! {
pub trait WARCallback<'a, T> = FnOnce(
pub trait WithAtomRecordCallback<'a, T> = AsyncFnOnce(
Box<dyn AtomDynfo>,
SysCtx,
AtomTypeId,
&'a [u8]
) -> LocalBoxFuture<'a, T>
) -> T
}
pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
get_sys_ctx: &impl Fn(api::SysId, ReqNot<api::ExtMsgSet>) -> F,
reqnot: ReqNot<api::ExtMsgSet>,
get_sys_ctx: &impl Fn(api::SysId) -> F,
atom: &'a api::Atom,
cb: impl WARCallback<'a, T>,
cb: impl WithAtomRecordCallback<'a, T>,
) -> T {
let mut data = &atom.data[..];
let ctx = get_sys_ctx(atom.owner, reqnot).await;
let inst = ctx.cted.inst();
let ctx = get_sys_ctx(atom.owner).await;
let inst = ctx.get::<CtedObj>().inst();
let id = AtomTypeId::decode(Pin::new(&mut data)).await;
let atom_record = atom_by_idx(inst.card(), id.clone()).expect("Atom ID reserved");
cb(atom_record, ctx, id, data).await
}
// pub fn extension_main(data: ExtensionData) {
// if thread::Builder::new()
// .name(format!("ext-main:{}", data.name))
// .spawn(|| extension_main_logic(data))
// .unwrap()
// .join()
// .is_err()
// {
// process::exit(-1)
// }
// }
pub struct ExtensionOwner {
rn: ReqNot<api::ExtMsgSet>,
_interner_cell: Rc<RefCell<Option<Interner>>>,
_systems_lock: Rc<Mutex<HashMap<api::SysId, SystemRecord>>>,
out_recv: Receiver<Vec<u8>>,
out_send: Sender<Vec<u8>>,
}
impl ExtPort for ExtensionOwner {
fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
self.rn.receive(msg).boxed_local()
Box::pin(async { self.out_send.send(msg.to_vec()).boxed_local().await.unwrap() })
}
fn recv<'a>(
&'a self,
cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
) -> LocalBoxFuture<'a, ()> {
async {
let msg = self.out_recv.recv().await.unwrap();
cb(&msg[..]).await
}
.boxed_local()
fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
Box::pin(async { (self.out_recv.recv().await).ok() })
}
}
pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
let api::HostHeader { log_strategy, msg_logs } =
api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await;
let mut buf = Vec::new();
pub fn extension_init(
data: ExtensionData,
host_header: api::HostHeader,
spawner: Spawner,
) -> ExtInit {
let api::HostHeader { log_strategy, msg_logs } = host_header;
let decls = (data.systems.iter().enumerate())
.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
.collect_vec();
let systems = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() }
.encode(Pin::new(&mut buf))
.await;
std::io::stdout().write_all(&buf).unwrap();
std::io::stdout().flush().unwrap();
let exiting = Arc::new(AtomicBool::new(false));
let systems_lock = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
let ext_header = api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() };
let (out_send, in_recv) = channel::bounded::<Vec<u8>>(1);
let (in_send, out_recv) = channel::bounded::<Vec<u8>>(1);
let (exit_send, exit_recv) = channel::bounded(1);
let logger = Logger::new(log_strategy);
let msg_logger = Logger::new(msg_logs);
let interner_cell = Rc::new(RefCell::new(None::<Rc<Interner>>));
let interner_cell = Rc::new(RefCell::new(None::<Interner>));
let interner_weak = Rc::downgrade(&interner_cell);
let obj_store = ObjStore::default();
let mk_ctx = clone!(
logger, systems, spawner, obj_store, interner_weak;
move |id: api::SysId, reqnot: ReqNot<api::ExtMsgSet>| {
clone!(logger, systems, spawner, obj_store, interner_weak; async move {
let cted = systems.lock().await[&id].cted.clone();
let interner_cell = (interner_weak.upgrade())
.expect("mk_ctx called after Interner rc dropped");
let i = (interner_cell.borrow().clone())
.expect("mk_ctx called before interner initialized");
SysCtx { id, cted, logger, reqnot, spawner, obj_store, i: i.clone() }
}.boxed_local())
});
let systems_weak = Rc::downgrade(&systems_lock);
let get_ctx = clone!(systems_weak; move |id: api::SysId| clone!(systems_weak; async move {
let systems =
systems_weak.upgrade().expect("System table dropped before request processing done");
systems.lock().await.get(&id).expect("System not found").ctx.clone()
}));
let init_ctx = {
clone!(interner_weak, spawner, logger);
move |id: api::SysId, cted: CtedObj, reqnot: ReqNot<ExtMsgSet>| {
clone!(interner_weak, spawner, logger; async move {
let interner_rc =
interner_weak.upgrade().expect("System construction order while shutting down");
let i = interner_rc.borrow().clone().expect("mk_ctx called very early, no interner!");
SysCtx::new(id, i, reqnot, spawner, logger, cted)
})
}
};
let rn = ReqNot::<api::ExtMsgSet>::new(
msg_logger.clone(),
move |a, _| async move { send_parent_msg(a).await.unwrap() }.boxed_local(),
clone!(systems, exiting, mk_ctx; move |n, reqnot| {
clone!(systems, exiting, mk_ctx; async move {
move |a, _| clone!(in_send; Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })),
clone!(systems_weak, exit_send, get_ctx; move |n, _| {
clone!(systems_weak, exit_send, get_ctx; async move {
match n {
api::HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
mem::drop(systems.lock().await.remove(&sys_id)),
if let Some(rc) = systems_weak.upgrade() {
mem::drop(rc.lock().await.remove(&sys_id))
},
api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
let ctx = mk_ctx(sys_id, reqnot).await;
let ctx = get_ctx(sys_id).await;
take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
}
}
}.boxed_local())
}),
{
clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
move |hand, req| {
clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
async move {
let interner_cell = interner_weak.upgrade().expect("Interner dropped before request");
let i = interner_cell.borrow().clone().expect("Request arrived before interner set");
@@ -190,102 +167,70 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
let (sys_id, _) = (decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system))
.expect("NewSystem call received for invalid system");
let cted = data.systems[sys_id].new_system(&new_sys);
let mut vfses = HashMap::new();
let lex_filter =
cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
});
let lazy_mems = Mutex::new(HashMap::new());
let rules = Mutex::new(HashMap::new());
let ctx = SysCtx {
cted: cted.clone(),
id: new_sys.id,
logger: logger.clone(),
reqnot: hand.reqnot(),
i: i.clone(),
obj_store: obj_store.clone(),
spawner: spawner.clone(),
};
let ctx = init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await;
let const_root = stream::from_iter(cted.inst().dyn_env())
.then(|(k, v)| {
let (req, lazy_mems, rules) = (&hand, &lazy_mems, &rules);
.then(|mem| {
let (req, lazy_mems) = (&hand, &lazy_mems);
clone!(i, ctx; async move {
let name = i.i(&k).await.to_api();
let value = v.into_api(&mut TIACtxImpl {
let mut tia_ctx = TreeIntoApiCtxImpl {
lazy_members: &mut *lazy_mems.lock().await,
rules: &mut *rules.lock().await,
sys: ctx,
basepath: &[],
path: Substack::Bottom,
req
})
.await;
(name, value)
};
(i.i(&mem.name).await.to_api(), mem.kind.into_api(&mut tia_ctx).await)
})
})
.collect()
.await;
let declfs = cted.inst().dyn_vfs().to_api_rec(&mut vfses, &i).await;
let record = SystemRecord {
declfs,
vfses,
cted,
lazy_members: lazy_mems.into_inner(),
rules: rules.into_inner(),
};
let record = SystemRecord { ctx, lazy_members: lazy_mems.into_inner() };
let systems = systems_weak.upgrade().expect("System constructed during shutdown");
systems.lock().await.insert(new_sys.id, record);
hand
.handle(&new_sys, &api::SystemInst { lex_filter, const_root, line_types: vec![] })
.await
let response = api::NewSystemResponse { lex_filter, const_root, line_types: vec![] };
hand.handle(&new_sys, &response).await
},
api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) => {
let sys_ctx = mk_ctx(sys_id, hand.reqnot()).await;
let sys_ctx = get_ctx(sys_id).await;
let systems = systems_weak.upgrade().expect("Member queried during shutdown");
let mut systems_g = systems.lock().await;
let SystemRecord { lazy_members, rules, .. } =
let SystemRecord { lazy_members, .. } =
systems_g.get_mut(&sys_id).expect("System not found");
let (path, cb) = match lazy_members.insert(tree_id, MemberRecord::Res) {
None => panic!("Tree for ID not found"),
Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
Some(MemberRecord::Gen(path, cb)) => (path, cb),
};
let tree = cb.build(Sym::new(path.clone(), &i).await.unwrap()).await;
let mut tia_ctx = TIACtxImpl {
let tree = cb.build(Sym::new(path.clone(), &i).await.unwrap(), sys_ctx.clone()).await;
let mut tia_ctx = TreeIntoApiCtxImpl {
sys: sys_ctx,
path: Substack::Bottom,
basepath: &path,
lazy_members,
rules,
req: &hand,
};
hand.handle(&get_tree, &tree.into_api(&mut tia_ctx).await).await
},
api::HostExtReq::VfsReq(api::VfsReq::GetVfs(get_vfs @ api::GetVfs(sys_id))) => {
let systems_g = systems.lock().await;
hand.handle(&get_vfs, &systems_g[&sys_id].declfs).await
},
api::HostExtReq::SysReq(api::SysReq::SysFwded(fwd)) => {
let api::SysFwded(sys_id, payload) = fwd;
let ctx = mk_ctx(sys_id, hand.reqnot()).await;
let sys = ctx.cted.inst();
let ctx = get_ctx(sys_id).await;
let sys = ctx.cted().inst();
sys.dyn_request(hand, payload).await
},
api::HostExtReq::VfsReq(api::VfsReq::VfsRead(vfs_read)) => {
let api::VfsRead(sys_id, vfs_id, path) = &vfs_read;
let ctx = mk_ctx(*sys_id, hand.reqnot()).await;
let systems_g = systems.lock().await;
let path = join_all(path.iter().map(|t| Tok::from_api(*t, &i))).await;
let vfs = systems_g[sys_id].vfses[vfs_id].load(&path, ctx).await;
hand.handle(&vfs_read, &vfs).await
},
api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, text, pos, id }) => {
let systems_g = systems.lock().await;
let lexers = systems_g[&sys].cted.inst().dyn_lexers();
mem::drop(systems_g);
api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) => {
let sys_ctx = get_ctx(sys).await;
let text = Tok::from_api(text, &i).await;
let ctx = LexContext { sys, id, pos, reqnot: hand.reqnot(), text: &text, i: &i };
let src = Sym::from_api(src, sys_ctx.i()).await;
let ctx = LexContext { id, pos, text: &text, src, ctx: sys_ctx.clone() };
let trigger_char = text.chars().nth(pos as usize).unwrap();
let err_na = err_not_applicable(&i).await;
let err_cascade = err_cascade(&i).await;
let lexers = sys_ctx.cted().inst().dyn_lexers();
for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char)) {
match lx.lex(&text[pos as usize..], &ctx).await {
Err(e) if e.any(|e| *e == err_na) => continue,
@@ -294,12 +239,7 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
return hand.handle(&lex, &eopt).await;
},
Ok((s, expr)) => {
let ctx = mk_ctx(sys, hand.reqnot()).await;
let expr = expr
.to_api(&mut |f, r| {
clone!(ctx; async move { do_extra(f, r, ctx).await }).boxed_local()
})
.await;
let expr = expr.into_api(&mut (), &mut (sys_ctx, &hand)).await;
let pos = (text.len() - s.len()) as u32;
return hand.handle(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
},
@@ -309,122 +249,104 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
hand.handle(&lex, &None).await
},
api::HostExtReq::ParseLine(pline) => {
let api::ParseLine { exported, comments, sys, line } = &pline;
let mut ctx = mk_ctx(*sys, hand.reqnot()).await;
let parsers = ctx.cted.inst().dyn_parsers();
let comments = join_all(comments.iter().map(|c| Comment::from_api(c, &i))).await;
let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx, &i).await;
let snip = Snippet::new(line.first().expect("Empty line"), &line, &i);
let api::ParseLine { module, src, exported, comments, sys, line } = &pline;
let mut ctx = get_ctx(*sys).await;
let parsers = ctx.cted().inst().dyn_parsers();
let src = Sym::from_api(*src, ctx.i()).await;
let comments =
join_all(comments.iter().map(|c| Comment::from_api(c, src.clone(), &i))).await;
let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx, &mut (), &src, &i).await;
let snip = Snippet::new(line.first().expect("Empty line"), &line);
let (head, tail) = snip.pop_front().unwrap();
let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") };
let parser =
parsers.iter().find(|p| p.line_head() == **name).expect("No parser candidate");
let o_line = match parser.parse(*exported, comments, tail) {
let module = Sym::from_api(*module, ctx.i()).await;
let pctx = ParsCtx::new(ctx.clone(), module);
let o_line = match parser.parse(pctx, *exported, comments, tail).await {
Err(e) => Err(e.to_api()),
Ok(t) => Ok(
ttv_to_api(t, &mut |f, range| {
clone!(ctx);
async move {
api::TokenTree { range, token: api::Token::Atom(f.clone().build(ctx).await) }
}
.boxed_local()
})
.await,
),
Ok(t) => Ok(linev_into_api(t, ctx.clone(), &hand).await),
};
hand.handle(&pline, &o_line).await
},
api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst { id, sys }) => {
let ctx = get_ctx(sys).await;
let cnst = get_const(id, ctx.clone()).await;
hand.handle(fpc, &cnst.api_return(ctx, &hand).await).await
},
api::HostExtReq::AtomReq(atom_req) => {
let atom = atom_req.get_atom();
let atom_req = atom_req.clone();
with_atom_record(&mk_ctx, hand.reqnot(), atom, move |nfo, ctx, id, buf| {
async move {
let actx = AtomCtx(buf, atom.drop, ctx.clone());
match &atom_req {
api::AtomReq::SerializeAtom(ser) => {
let mut buf = enc_vec(&id).await;
let refs_opt = nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await;
hand.handle(ser, &refs_opt.map(|refs| (buf, refs))).await
},
api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
hand.handle(print, &nfo.print(actx).await.to_api()).await,
api::AtomReq::Fwded(fwded) => {
let api::Fwded(_, key, payload) = &fwded;
let mut reply = Vec::new();
let key = Sym::from_api(*key, &i).await;
let some = nfo
.handle_req(
actx,
key,
Pin::<&mut &[u8]>::new(&mut &payload[..]),
Pin::<&mut Vec<_>>::new(&mut reply),
)
.await;
hand.handle(fwded, &some.then_some(reply)).await
},
api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
let ret = nfo.call_ref(actx, *arg).await;
hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
},
api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
let ret = nfo.call(actx, *arg).await;
hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
},
api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
Ok(opt) => match opt {
None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
Some(cont) => {
let cont = cont.api_return(ctx.clone(), &hand).await;
hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
},
with_atom_record(&get_ctx, atom, async move |nfo, ctx, id, buf| {
let actx = AtomCtx(buf, atom.drop, ctx.clone());
match &atom_req {
api::AtomReq::SerializeAtom(ser) => {
let mut buf = enc_vec(&id).await;
match nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await {
None => hand.handle(ser, &None).await,
Some(refs) => {
let refs =
join_all(refs.into_iter().map(|ex| async { ex.into_api(&mut ()).await }))
.await;
hand.handle(ser, &Some((buf, refs))).await
},
}
},
api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
hand.handle(print, &nfo.print(actx).await.to_api()).await,
api::AtomReq::Fwded(fwded) => {
let api::Fwded(_, key, payload) = &fwded;
let mut reply = Vec::new();
let key = Sym::from_api(*key, &i).await;
let some = nfo
.handle_req(
actx,
key,
Pin::<&mut &[u8]>::new(&mut &payload[..]),
Pin::<&mut Vec<_>>::new(&mut reply),
)
.await;
hand.handle(fwded, &some.then_some(reply)).await
},
api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
// SAFETY: function calls own their argument implicitly
let expr_handle = unsafe { ExprHandle::from_args(ctx.clone(), *arg) };
let ret = nfo.call_ref(actx, Expr::from_handle(Rc::new(expr_handle))).await;
hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
},
api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
// SAFETY: function calls own their argument implicitly
let expr_handle = unsafe { ExprHandle::from_args(ctx.clone(), *arg) };
let ret = nfo.call(actx, Expr::from_handle(Rc::new(expr_handle))).await;
hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
},
api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
Ok(opt) => match opt {
None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
Some(cont) => {
let cont = cont.api_return(ctx.clone(), &hand).await;
hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
},
},
}
},
}
.boxed_local()
})
.await
},
api::HostExtReq::DeserAtom(deser) => {
let api::DeserAtom(sys, buf, refs) = &deser;
let mut read = &mut &buf[..];
let ctx = mk_ctx(*sys, hand.reqnot()).await;
let ctx = get_ctx(*sys).await;
// SAFETY: deserialization implicitly grants ownership to previously owned exprs
let refs = (refs.iter())
.map(|tk| unsafe { ExprHandle::from_args(ctx.clone(), *tk) })
.map(|handle| Expr::from_handle(Rc::new(handle)))
.collect_vec();
let id = AtomTypeId::decode(Pin::new(&mut read)).await;
let inst = ctx.cted.inst();
let inst = ctx.cted().inst();
let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, refs).await).await
},
orchid_api::HostExtReq::ApplyMacro(am) => {
let tok = hand.will_handle_as(&am);
let ApplyMacro { id, params, run_id, sys } = am;
let sys_ctx = mk_ctx(sys, hand.reqnot()).await;
let mut ctx =
RuleCtx { args: ahash::HashMap::default(), run_id, sys: sys_ctx.clone() };
for (k, v) in params {
ctx.args.insert(
Tok::from_api(k, &i).await,
mtreev_from_api(&v, &i, &mut |_| panic!("No atom in macro prompt!")).await,
);
}
let err_cascade = err_cascade(&i).await;
let systems_g = systems.lock().await;
let rule = &systems_g[&sys].rules[&id];
match (rule.apply)(ctx).await {
Err(e) => {
let new_errors = e.keep_only(|e| *e != err_cascade);
hand.handle_as(tok, &new_errors.map(|e| Err(e.to_api()))).await
},
Ok(t) => {
let result = mtreev_to_api(&t, &mut |a| {
clone!(sys_ctx; async move {
api::MacroToken::Atom(a.clone().build(sys_ctx.clone()).await)
}.boxed_local())
})
.await;
hand.handle_as(tok, &Some(Ok(result))).await
},
}
hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, &refs).await).await
},
}
}
@@ -432,9 +354,24 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
}
},
);
*interner_cell.borrow_mut() = Some(Rc::new(Interner::new_replica(rn.clone().map())));
while !exiting.load(Ordering::Relaxed) {
let rcvd = recv_parent_msg().await.unwrap();
spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd).await })))
*interner_cell.borrow_mut() =
Some(Interner::new_replica(rn.clone().map(|ir: IntReq| ir.into_root())));
spawner(Box::pin(clone!(spawner; async move {
let mut streams = stream_select! { in_recv.map(Some), exit_recv.map(|_| None) };
while let Some(item) = streams.next().await {
match item {
Some(rcvd) => spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd[..]).await }))),
None => break,
}
}
})));
ExtInit {
header: ext_header,
port: Box::new(ExtensionOwner {
out_recv,
out_send,
_interner_cell: interner_cell,
_systems_lock: systems_lock,
}),
}
}

34
orchid-extension/src/expr.rs
View File

@@ -20,12 +20,20 @@ pub struct ExprHandle {
pub ctx: SysCtx,
}
impl ExprHandle {
pub(crate) fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
/// # Safety
///
/// This function does not signal to take ownership of the expr. It must only
/// be called on tickets that are already implicitly owned.
pub unsafe fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() }
pub async fn clone(&self) -> Self {
self.ctx.reqnot.notify(api::Acquire(self.ctx.id, self.tk)).await;
self.ctx.reqnot().notify(api::Acquire(self.ctx.sys_id(), self.tk)).await;
Self { ctx: self.ctx.clone(), tk: self.tk }
}
/// Drop the handle and get the ticket without a release notification.
/// Use this with messages that imply ownership transfer. This function is
/// safe because abusing it is a memory leak.
pub fn into_tk(self) -> api::ExprTicket { self.destructure().0 }
}
impl fmt::Debug for ExprHandle {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -34,9 +42,9 @@ impl fmt::Debug for ExprHandle {
}
impl Drop for ExprHandle {
fn drop(&mut self) {
let notif = api::Release(self.ctx.id, self.tk);
let SysCtx { reqnot, spawner, .. } = self.ctx.clone();
spawner(Box::pin(async move { reqnot.notify(notif).await }))
let notif = api::Release(self.ctx.sys_id(), self.tk);
let reqnot = self.ctx.reqnot().clone();
self.ctx.spawner()(Box::pin(async move { reqnot.notify(notif).await }))
}
}
@@ -53,20 +61,20 @@ impl Expr {
pub async fn data(&self) -> &ExprData {
(self.data.get_or_init(async {
let details = self.handle.ctx.reqnot.request(api::Inspect { target: self.handle.tk }).await;
let pos = Pos::from_api(&details.location, &self.handle.ctx.i).await;
let details = self.handle.ctx.reqnot().request(api::Inspect { target: self.handle.tk }).await;
let pos = Pos::from_api(&details.location, self.handle.ctx.i()).await;
let kind = match details.kind {
api::InspectedKind::Atom(a) =>
ExprKind::Atom(ForeignAtom::new(self.handle.clone(), a, pos.clone())),
api::InspectedKind::Bottom(b) =>
ExprKind::Bottom(OrcErrv::from_api(&b, &self.handle.ctx.i).await),
ExprKind::Bottom(OrcErrv::from_api(&b, self.handle.ctx.i()).await),
api::InspectedKind::Opaque => ExprKind::Opaque,
};
ExprData { pos, kind }
}))
.await
}
pub async fn atom(self) -> Result<ForeignAtom<'static>, Self> {
pub async fn atom(self) -> Result<ForeignAtom, Self> {
match self.data().await {
ExprData { kind: ExprKind::Atom(atom), .. } => Ok(atom.clone()),
_ => Err(self),
@@ -75,7 +83,9 @@ impl Expr {
pub fn handle(&self) -> Rc<ExprHandle> { self.handle.clone() }
pub fn ctx(&self) -> SysCtx { self.handle.ctx.clone() }
pub fn gen(&self) -> GExpr { GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(self.clone()) } }
pub fn slot(&self) -> GExpr {
GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(self.clone()) }
}
}
impl Format for Expr {
async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -83,7 +93,7 @@ impl Format for Expr {
ExprKind::Opaque => "OPAQUE".to_string().into(),
ExprKind::Bottom(b) => format!("Bottom({b})").into(),
ExprKind::Atom(a) =>
FmtUnit::from_api(&self.handle.ctx.reqnot.request(ExtAtomPrint(a.atom.clone())).await),
FmtUnit::from_api(&self.handle.ctx.reqnot().request(ExtAtomPrint(a.atom.clone())).await),
}
}
}
@@ -96,7 +106,7 @@ pub struct ExprData {
#[derive(Clone, Debug)]
pub enum ExprKind {
Atom(ForeignAtom<'static>),
Atom(ForeignAtom),
Bottom(OrcErrv),
Opaque,
}

46
orchid-extension/src/fs.rs
View File

@@ -1,46 +0,0 @@
use std::num::NonZero;
use futures::FutureExt;
use futures::future::LocalBoxFuture;
use hashbrown::HashMap;
use orchid_base::interner::{Interner, Tok};
use crate::api;
use crate::system::SysCtx;
pub trait VirtFS: Send + Sync + 'static {
fn load<'a>(
&'a self,
path: &'a [Tok<String>],
ctx: SysCtx,
) -> LocalBoxFuture<'a, api::OrcResult<api::Loaded>>;
}
pub enum DeclFs {
Lazy(&'static dyn VirtFS),
Mod(&'static [(&'static str, DeclFs)]),
}
impl DeclFs {
pub async fn to_api_rec(
&self,
vfses: &mut HashMap<api::VfsId, &'static dyn VirtFS>,
i: &Interner,
) -> api::EagerVfs {
match self {
DeclFs::Lazy(fs) => {
let vfsc: u16 = vfses.len().try_into().expect("too many vfses (more than u16::MAX)");
let id = api::VfsId(NonZero::new(vfsc + 1).unwrap());
vfses.insert(id, *fs);
api::EagerVfs::Lazy(id)
},
DeclFs::Mod(children) => {
let mut output = std::collections::HashMap::new();
for (k, v) in children.iter() {
output
.insert(i.i::<String>(*k).await.to_api(), v.to_api_rec(vfses, i).boxed_local().await);
}
api::EagerVfs::Eager(output)
},
}
}
}

50
orchid-extension/src/func_atom.rs
View File

@@ -13,16 +13,16 @@ use never::Never;
use orchid_api_traits::Encode;
use orchid_base::clone;
use orchid_base::error::OrcRes;
use orchid_base::format::{FmtCtxImpl, Format, take_first};
use orchid_base::format::{FmtCtx, FmtUnit};
use orchid_base::name::Sym;
use trait_set::trait_set;
use crate::atom::{Atomic, MethodSetBuilder};
use crate::atom::Atomic;
use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
use crate::conv::ToExpr;
use crate::expr::{Expr, ExprHandle};
use crate::expr::Expr;
use crate::gen_expr::GExpr;
use crate::system::SysCtx;
use crate::system::{SysCtx, SysCtxEntry};
trait_set! {
trait FunCB = Fn(Vec<Expr>) -> LocalBoxFuture<'static, OrcRes<GExpr>> + 'static;
@@ -33,9 +33,9 @@ pub trait ExprFunc<I, O>: Clone + 'static {
fn apply(&self, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
}
thread_local! {
static FUNS: Rc<Mutex<HashMap<Sym, (u8, Rc<dyn FunCB>)>>> = Rc::default();
}
#[derive(Default)]
struct FunsCtx(Mutex<HashMap<Sym, (u8, Rc<dyn FunCB>)>>);
impl SysCtxEntry for FunsCtx {}
/// An Atom representing a partially applied named native function. These
/// partial calls are serialized into the name of the native function and the
@@ -50,9 +50,9 @@ pub(crate) struct Fun {
fun: Rc<dyn FunCB>,
}
impl Fun {
pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, f: F) -> Self {
let funs = FUNS.with(|funs| funs.clone());
let mut fung = funs.lock().await;
pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, ctx: SysCtx, f: F) -> Self {
let funs: &FunsCtx = ctx.get_or_default();
let mut fung = funs.0.lock().await;
let fun = if let Some(x) = fung.get(&path) {
x.1.clone()
} else {
@@ -67,14 +67,13 @@ impl Fun {
impl Atomic for Fun {
type Data = ();
type Variant = OwnedVariant;
fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
}
impl OwnedAtom for Fun {
type Refs = Vec<Expr>;
async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
async fn call_ref(&self, arg: ExprHandle) -> GExpr {
async fn call_ref(&self, arg: Expr) -> GExpr {
std::io::Write::flush(&mut std::io::stderr()).unwrap();
let new_args = self.args.iter().cloned().chain([Expr::from_handle(Rc::new(arg))]).collect_vec();
let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
if new_args.len() == self.arity.into() {
(self.fun)(new_args).await.to_expr()
} else {
@@ -82,18 +81,18 @@ impl OwnedAtom for Fun {
.to_expr()
}
}
async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await }
async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
self.path.to_api().encode(write).await;
self.args.clone()
}
async fn deserialize(mut ctx: impl DeserializeCtx, args: Self::Refs) -> Self {
let sys = ctx.sys();
let path = Sym::from_api(ctx.decode().await, &sys.i).await;
let (arity, fun) = FUNS.with(|f| f.clone()).lock().await.get(&path).unwrap().clone();
let path = Sym::from_api(ctx.decode().await, sys.i()).await;
let (arity, fun) = sys.get_or_default::<FunsCtx>().0.lock().await.get(&path).unwrap().clone();
Self { args, arity, path, fun }
}
async fn print(&self, _: SysCtx) -> orchid_base::format::FmtUnit {
async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
format!("{}:{}/{}", self.path, self.args.len(), self.arity).into()
}
}
@@ -117,25 +116,22 @@ impl Lambda {
impl Atomic for Lambda {
type Data = ();
type Variant = OwnedVariant;
fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
}
impl OwnedAtom for Lambda {
type Refs = Never;
async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
async fn call_ref(&self, arg: ExprHandle) -> GExpr {
let new_args = self.args.iter().cloned().chain([Expr::from_handle(Rc::new(arg))]).collect_vec();
async fn call_ref(&self, arg: Expr) -> GExpr {
let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
if new_args.len() == self.arity.into() {
(self.fun)(new_args).await.to_expr()
} else {
Self { args: new_args, arity: self.arity, fun: self.fun.clone() }.to_expr()
}
}
async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await }
async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
}
mod expr_func_derives {
use std::future::Future;
use orchid_base::error::OrcRes;
use super::ExprFunc;
@@ -145,12 +141,12 @@ mod expr_func_derives {
macro_rules! expr_func_derive {
($arity: tt, $($t:ident),*) => {
paste::paste!{
pastey::paste!{
impl<
$($t: TryFromExpr, )*
Fut: Future<Output: ToExpr>,
Func: Fn($($t,)*) -> Fut + Clone + Send + Sync + 'static
> ExprFunc<($($t,)*), Fut::Output> for Func {
Out: ToExpr,
Func: AsyncFn($($t,)*) -> Out + Clone + Send + Sync + 'static
> ExprFunc<($($t,)*), Out> for Func {
const ARITY: u8 = $arity;
async fn apply(&self, v: Vec<Expr>) -> OrcRes<GExpr> {
assert_eq!(v.len(), Self::ARITY.into(), "Arity mismatch");

38
orchid-extension/src/gen_expr.rs
View File

@@ -1,11 +1,12 @@
use std::future::Future;
use std::rc::Rc;
use futures::FutureExt;
use orchid_base::error::{OrcErr, OrcErrv};
use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
use orchid_base::location::Pos;
use orchid_base::match_mapping;
use orchid_base::name::Sym;
use orchid_base::reqnot::ReqHandlish;
use orchid_base::{match_mapping, tl_cache};
use crate::api;
use crate::atom::{AtomFactory, ToAtom};
@@ -14,6 +15,7 @@ use crate::expr::Expr;
use crate::func_atom::Lambda;
use crate::system::SysCtx;
#[derive(Clone, Debug)]
pub struct GExpr {
pub kind: GExprKind,
pub pos: Pos,
@@ -34,7 +36,13 @@ impl GExpr {
}
}
}
impl Format for GExpr {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
self.kind.print(c).await
}
}
#[derive(Clone, Debug)]
pub enum GExprKind {
Call(Box<GExpr>, Box<GExpr>),
Lambda(u64, Box<GExpr>),
@@ -67,6 +75,28 @@ impl GExprKind {
})
}
}
impl Format for GExprKind {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
match self {
GExprKind::Call(f, x) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} ({1})")))
.units([f.print(c).await, x.print(c).await]),
GExprKind::Lambda(arg, body) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0}.{1}")))
.units([arg.to_string().into(), body.print(c).await]),
GExprKind::Arg(arg) => arg.to_string().into(),
GExprKind::Seq(a, b) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0}] {1}")))
.units([a.print(c).await, b.print(c).await]),
GExprKind::Const(sym) => sym.to_string().into(),
GExprKind::NewAtom(atom_factory) => atom_factory.to_string().into(),
GExprKind::Slot(expr) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{{{0}}}")))
.units([expr.print(c).await]),
GExprKind::Bottom(orc_errv) => orc_errv.to_string().into(),
}
}
}
fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
@@ -100,9 +130,9 @@ pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> GExpr {
inherit(GExprKind::Bottom(OrcErrv::new(ev).unwrap()))
}
pub fn with<I: TryFromExpr, Fut: Future<Output: ToExpr>>(
pub fn with<I: TryFromExpr, O: ToExpr>(
expr: GExpr,
cont: impl Fn(I) -> Fut + Clone + Send + Sync + 'static,
cont: impl AsyncFn(I) -> O + Clone + Send + Sync + 'static,
) -> GExpr {
call([lambda(0, [seq([arg(0), call([Lambda::new(cont).to_expr(), arg(0)])])]), expr])
}

44
orchid-extension/src/lexer.rs
View File

@@ -1,16 +1,18 @@
use std::fmt;
use std::future::Future;
use std::ops::{Range, RangeInclusive};
use std::ops::RangeInclusive;
use futures::FutureExt;
use futures::future::LocalBoxFuture;
use orchid_base::error::{OrcErr, OrcRes, mk_err};
use orchid_base::interner::{Interner, Tok};
use orchid_base::location::Pos;
use orchid_base::reqnot::{ReqNot, Requester};
use orchid_base::tree::TokHandle;
use orchid_base::location::{Pos, SrcRange};
use orchid_base::name::Sym;
use orchid_base::reqnot::Requester;
use crate::api;
use crate::tree::{GenTok, GenTokTree};
use crate::system::SysCtx;
use crate::tree::GenTokTree;
pub async fn err_cascade(i: &Interner) -> OrcErr {
mk_err(
@@ -30,27 +32,33 @@ pub async fn err_not_applicable(i: &Interner) -> OrcErr {
}
pub struct LexContext<'a> {
pub ctx: SysCtx,
pub text: &'a Tok<String>,
pub sys: api::SysId,
pub id: api::ParsId,
pub pos: u32,
pub reqnot: ReqNot<api::ExtMsgSet>,
pub i: &'a Interner,
pub src: Sym,
}
impl<'a> LexContext<'a> {
pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree<'a>)> {
pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree)> {
let start = self.pos(tail);
let Some(lx) = self.reqnot.request(api::SubLex { pos: start, id: self.id }).await else {
return Err(err_cascade(self.i).await.into());
let Some(lx) = self.ctx.reqnot().request(api::SubLex { pos: start, id: self.id }).await else {
return Err(err_cascade(self.ctx.i()).await.into());
};
Ok((&self.text[lx.pos as usize..], GenTok::Slot(TokHandle::new(lx.ticket)).at(start..lx.pos)))
let tree =
GenTokTree::from_api(&lx.tree, &mut self.ctx.clone(), &mut (), &self.src, self.ctx.i()).await;
Ok((&self.text[lx.pos as usize..], tree))
}
pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
pub fn tok_ran(&self, len: u32, tail: &'a str) -> Range<u32> {
self.pos(tail) - len..self.pos(tail)
pub fn pos_tt(&self, tail_with: &'a str, tail_without: &'a str) -> SrcRange {
SrcRange::new(self.pos(tail_with)..self.pos(tail_without), &self.src)
}
pub fn pos_lt(&self, len: impl TryInto<u32, Error: fmt::Debug>, tail: &'a str) -> SrcRange {
SrcRange::new(self.pos(tail) - len.try_into().unwrap()..self.pos(tail), &self.src)
}
pub fn i(&self) -> &Interner { self.ctx.i() }
}
pub trait Lexer: Send + Sync + Sized + Default + 'static {
@@ -58,7 +66,7 @@ pub trait Lexer: Send + Sync + Sized + Default + 'static {
fn lex<'a>(
tail: &'a str,
ctx: &'a LexContext<'a>,
) -> impl Future<Output = OrcRes<(&'a str, GenTokTree<'a>)>>;
) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
}
pub trait DynLexer: Send + Sync + 'static {
@@ -67,7 +75,7 @@ pub trait DynLexer: Send + Sync + 'static {
&self,
tail: &'a str,
ctx: &'a LexContext<'a>,
) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree<'a>)>>;
) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>>;
}
impl<T: Lexer> DynLexer for T {
@@ -76,7 +84,7 @@ impl<T: Lexer> DynLexer for T {
&self,
tail: &'a str,
ctx: &'a LexContext<'a>,
) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree<'a>)>> {
) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>> {
T::lex(tail, ctx).boxed_local()
}
}

3
orchid-extension/src/lib.rs
View File

@@ -6,14 +6,13 @@ pub mod atom_thin;
pub mod conv;
pub mod entrypoint;
pub mod expr;
pub mod fs;
pub mod func_atom;
pub mod gen_expr;
pub mod lexer;
pub mod macros;
pub mod msg;
pub mod other_system;
pub mod parser;
pub mod system;
pub mod system_ctor;
pub mod tokio;
pub mod tree;

102
orchid-extension/src/macros.rs
View File

@@ -1,102 +0,0 @@
use std::rc::Rc;
use ahash::HashMap;
use futures::future::{LocalBoxFuture, join_all};
use itertools::Itertools;
use never::Never;
use orchid_base::error::OrcRes;
use orchid_base::interner::Tok;
use orchid_base::macros::{MTree, mtreev_from_api, mtreev_to_api};
use orchid_base::reqnot::Requester;
use trait_set::trait_set;
use crate::api;
use crate::atom::AtomFactory;
use crate::lexer::err_cascade;
use crate::system::SysCtx;
use crate::tree::TreeIntoApiCtx;
pub trait Macro {
fn pattern() -> MTree<'static, Never>;
fn apply(binds: HashMap<Tok<String>, MTree<'_, Never>>) -> MTree<'_, AtomFactory>;
}
pub trait DynMacro {
fn pattern(&self) -> MTree<'static, Never>;
fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a, Never>>) -> MTree<'a, AtomFactory>;
}
impl<T: Macro> DynMacro for T {
fn pattern(&self) -> MTree<'static, Never> { Self::pattern() }
fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a, Never>>) -> MTree<'a, AtomFactory> {
Self::apply(binds)
}
}
pub struct RuleCtx<'a> {
pub(crate) args: HashMap<Tok<String>, Vec<MTree<'a, Never>>>,
pub(crate) run_id: api::ParsId,
pub(crate) sys: SysCtx,
}
impl<'a> RuleCtx<'a> {
pub async fn recurse(&mut self, tree: &[MTree<'a, Never>]) -> OrcRes<Vec<MTree<'a, Never>>> {
let req = api::RunMacros {
run_id: self.run_id,
query: mtreev_to_api(tree, &mut |b| match *b {}).await,
};
let Some(treev) = self.sys.reqnot.request(req).await else {
return Err(err_cascade(&self.sys.i).await.into());
};
static ATOM_MSG: &str = "Returned atom from Rule recursion";
Ok(mtreev_from_api(&treev, &self.sys.i, &mut |_| panic!("{ATOM_MSG}")).await)
}
pub fn getv(&mut self, key: &Tok<String>) -> Vec<MTree<'a, Never>> {
self.args.remove(key).expect("Key not found")
}
pub fn gets(&mut self, key: &Tok<String>) -> MTree<'a, Never> {
let v = self.getv(key);
assert!(v.len() == 1, "Not a scalar");
v.into_iter().next().unwrap()
}
pub fn unused_arg<'b>(&mut self, keys: impl IntoIterator<Item = &'b Tok<String>>) {
keys.into_iter().for_each(|k| {
self.getv(k);
});
}
}
trait_set! {
pub trait RuleCB = for<'a> Fn(RuleCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Vec<MTree<'a, AtomFactory>>>>;
}
pub struct Rule {
pub(crate) comments: Vec<String>,
pub(crate) pattern: Vec<MTree<'static, Never>>,
pub(crate) apply: Rc<dyn RuleCB>,
}
impl Rule {
pub(crate) async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MacroRule {
api::MacroRule {
comments: join_all(self.comments.iter().map(|c| async {
api::Comment { text: ctx.sys().i.i(c).await.to_api(), location: api::Location::Inherit }
}))
.await,
location: api::Location::Inherit,
pattern: mtreev_to_api(&self.pattern, &mut |b| match *b {}).await,
id: ctx.with_rule(Rc::new(self)),
}
}
}
pub fn rule_cmt<'a>(
cmt: impl IntoIterator<Item = &'a str>,
pattern: Vec<MTree<'static, Never>>,
apply: impl RuleCB + 'static,
) -> Rule {
let comments = cmt.into_iter().map(|s| s.to_string()).collect_vec();
Rule { comments, pattern, apply: Rc::new(apply) }
}
pub fn rule(pattern: Vec<MTree<'static, Never>>, apply: impl RuleCB + 'static) -> Rule {
rule_cmt([], pattern, apply)
}

159
orchid-extension/src/parser.rs
View File

@@ -1,40 +1,179 @@
use orchid_base::error::OrcRes;
use orchid_base::parse::{Comment, Snippet};
use std::marker::PhantomData;
use crate::atom::{AtomFactory, ForeignAtom};
use futures::FutureExt;
use futures::future::{LocalBoxFuture, join_all};
use itertools::Itertools;
use orchid_api::ResolveNames;
use orchid_base::error::OrcRes;
use orchid_base::id_store::IdStore;
use orchid_base::interner::Tok;
use orchid_base::location::SrcRange;
use orchid_base::name::Sym;
use orchid_base::parse::{Comment, Snippet};
use orchid_base::reqnot::{ReqHandlish, Requester};
use orchid_base::tree::ttv_into_api;
use crate::api;
use crate::expr::Expr;
use crate::gen_expr::GExpr;
use crate::system::{SysCtx, SysCtxEntry};
use crate::tree::GenTokTree;
pub type GenSnippet<'a> = Snippet<'a, 'a, ForeignAtom<'a>, AtomFactory>;
pub type GenSnippet<'a> = Snippet<'a, Expr, GExpr>;
pub trait Parser: Send + Sync + Sized + Default + 'static {
const LINE_HEAD: &'static str;
fn parse(
fn parse<'a>(
ctx: ParsCtx<'a>,
exported: bool,
comments: Vec<Comment>,
line: GenSnippet<'_>,
) -> OrcRes<Vec<GenTokTree<'_>>>;
line: GenSnippet<'a>,
) -> impl Future<Output = OrcRes<Vec<ParsedLine>>> + 'a;
}
pub trait DynParser: Send + Sync + 'static {
fn line_head(&self) -> &'static str;
fn parse<'a>(
&self,
ctx: ParsCtx<'a>,
exported: bool,
comments: Vec<Comment>,
line: GenSnippet<'a>,
) -> OrcRes<Vec<GenTokTree<'a>>>;
) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>>;
}
impl<T: Parser> DynParser for T {
fn line_head(&self) -> &'static str { Self::LINE_HEAD }
fn parse<'a>(
&self,
ctx: ParsCtx<'a>,
exported: bool,
comments: Vec<Comment>,
line: GenSnippet<'a>,
) -> OrcRes<Vec<GenTokTree<'a>>> {
Self::parse(exported, comments, line)
) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>> {
Box::pin(async move { Self::parse(ctx, exported, comments, line).await })
}
}
pub type ParserObj = &'static dyn DynParser;
pub struct ParsCtx<'a> {
_parse: PhantomData<&'a mut ()>,
ctx: SysCtx,
module: Sym,
}
impl ParsCtx<'_> {
pub(crate) fn new(ctx: SysCtx, module: Sym) -> Self { Self { _parse: PhantomData, ctx, module } }
pub fn ctx(&self) -> &SysCtx { &self.ctx }
pub fn module(&self) -> Sym { self.module.clone() }
}
type BoxConstCallback = Box<dyn FnOnce(ConstCtx) -> LocalBoxFuture<'static, GExpr>>;
#[derive(Default)]
struct ParsedConstCtxEntry {
consts: IdStore<BoxConstCallback>,
}
impl SysCtxEntry for ParsedConstCtxEntry {}
pub struct ParsedLine {
pub sr: SrcRange,
pub comments: Vec<Comment>,
pub kind: ParsedLineKind,
}
impl ParsedLine {
pub async fn into_api(self, ctx: SysCtx, hand: &dyn ReqHandlish) -> api::ParsedLine {
api::ParsedLine {
comments: self.comments.into_iter().map(|c| c.to_api()).collect(),
source_range: self.sr.to_api(),
kind: match self.kind {
ParsedLineKind::Mem(mem) => api::ParsedLineKind::Member(api::ParsedMember {
name: mem.name.to_api(),
exported: mem.exported,
kind: match mem.kind {
ParsedMemKind::Const(cb) => api::ParsedMemberKind::Constant(api::ParsedConstId(
ctx.get_or_default::<ParsedConstCtxEntry>().consts.add(cb).id(),
)),
ParsedMemKind::Mod(plv) =>
api::ParsedMemberKind::Module(linev_into_api(plv, ctx, hand).boxed_local().await),
},
}),
ParsedLineKind::Rec(tv) =>
api::ParsedLineKind::Recursive(ttv_into_api(tv, &mut (), &mut (ctx, hand)).await),
},
}
}
}
pub(crate) async fn linev_into_api(
v: Vec<ParsedLine>,
ctx: SysCtx,
hand: &dyn ReqHandlish,
) -> Vec<api::ParsedLine> {
join_all(v.into_iter().map(|l| l.into_api(ctx.clone(), hand))).await
}
pub enum ParsedLineKind {
Mem(ParsedMem),
Rec(Vec<GenTokTree>),
}
pub struct ParsedMem {
name: Tok<String>,
exported: bool,
kind: ParsedMemKind,
}
pub enum ParsedMemKind {
Const(BoxConstCallback),
Mod(Vec<ParsedLine>),
}
impl ParsedMemKind {
pub fn cnst<F: AsyncFnOnce(ConstCtx) -> GExpr + 'static>(f: F) -> Self {
Self::Const(Box::new(|ctx| Box::pin(f(ctx))))
}
}
/* TODO: how the macro runner uses the multi-stage loader
Since the macro runner actually has to invoke the interpreter,
it'll run at const-time and not at postparse-time anyway.
pasing stage establishes the role of every constant as a macro keyword
postparse / const load links up constants with every macro they can directly invoke
the constants representing the keywords might not actually be postparsed,
\ the connection is instead made by detecting in the macro system that the
\ resolved name is owned by a macro
the returned constant from this call is always an entrypoint call to
\ the macro system
the constants representing the keywords resolve to panic
execute relies on these links detected in the extension to dispatch relevant macros
*/
pub struct ConstCtx {
ctx: SysCtx,
constid: api::ParsedConstId,
}
impl ConstCtx {
pub async fn names<const N: usize>(&self, names: [&Sym; N]) -> [Option<Sym>; N] {
let resolve_names = ResolveNames {
constid: self.constid,
sys: self.ctx.sys_id(),
names: names.into_iter().map(|n| n.to_api()).collect_vec(),
};
let names = self.ctx.reqnot().request(resolve_names).await;
let mut results = [const { None }; N];
for (i, name) in names.into_iter().enumerate().filter_map(|(i, n)| Some((i, n?))) {
results[i] = Some(Sym::from_api(name, self.ctx.i()).await);
}
results
}
}
pub(crate) async fn get_const(id: api::ParsedConstId, ctx: SysCtx) -> GExpr {
let ent = ctx.get::<ParsedConstCtxEntry>();
let rec = ent.consts.get(id.0).expect("Bad ID or double read of parsed const");
let ctx = ConstCtx { constid: id, ctx: ctx.clone() };
rec.remove()(ctx).await
}

112
orchid-extension/src/system.rs
View File

@@ -1,12 +1,13 @@
use core::fmt;
use std::any::{TypeId, type_name};
use std::any::{Any, TypeId, type_name};
use std::fmt;
use std::future::Future;
use std::num::NonZero;
use std::pin::Pin;
use std::rc::Rc;
use futures::future::LocalBoxFuture;
use hashbrown::HashMap;
use memo_map::MemoMap;
use orchid_api::ExtMsgSet;
use orchid_api_traits::{Coding, Decode};
use orchid_base::boxed_iter::BoxedIter;
use orchid_base::builtin::Spawner;
@@ -16,14 +17,12 @@ use orchid_base::reqnot::{Receipt, ReqNot};
use crate::api;
use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TypAtom, get_info};
use crate::atom_owned::ObjStore;
use crate::entrypoint::ExtReq;
use crate::fs::DeclFs;
use crate::func_atom::Fun;
use crate::lexer::LexerObj;
use crate::parser::ParserObj;
use crate::system_ctor::{CtedObj, SystemCtor};
use crate::tree::MemKind;
use crate::tree::GenMember;
/// System as consumed by foreign code
pub trait SystemCard: Default + Send + Sync + 'static {
@@ -82,16 +81,14 @@ impl<T: SystemCard> DynSystemCard for T {
/// System as defined by author
pub trait System: Send + Sync + SystemCard + 'static {
fn env() -> Vec<(String, MemKind)>;
fn vfs() -> DeclFs;
fn env() -> Vec<GenMember>;
fn lexers() -> Vec<LexerObj>;
fn parsers() -> Vec<ParserObj>;
fn request(hand: ExtReq<'_>, req: Self::Req) -> impl Future<Output = Receipt<'_>>;
}
pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
fn dyn_env(&self) -> HashMap<String, MemKind>;
fn dyn_vfs(&self) -> DeclFs;
fn dyn_env(&self) -> Vec<GenMember>;
fn dyn_lexers(&self) -> Vec<LexerObj>;
fn dyn_parsers(&self) -> Vec<ParserObj>;
fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>>;
@@ -99,8 +96,7 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
}
impl<T: System> DynSystem for T {
fn dyn_env(&self) -> HashMap<String, MemKind> { Self::env().into_iter().collect() }
fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
fn dyn_env(&self) -> Vec<GenMember> { Self::env() }
fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>> {
@@ -111,16 +107,16 @@ impl<T: System> DynSystem for T {
fn card(&self) -> &dyn DynSystemCard { self }
}
pub async fn downcast_atom<A>(foreign: ForeignAtom<'_>) -> Result<TypAtom<'_, A>, ForeignAtom<'_>>
pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom>
where A: AtomicFeatures {
let mut data = &foreign.atom.data[..];
let ctx = foreign.ctx.clone();
let ctx = foreign.ctx().clone();
let value = AtomTypeId::decode(Pin::new(&mut data)).await;
let own_inst = ctx.cted.inst();
let owner = if ctx.id == foreign.atom.owner {
let own_inst = ctx.get::<CtedObj>().inst();
let owner = if *ctx.get::<api::SysId>() == foreign.atom.owner {
own_inst.card()
} else {
(ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
(ctx.get::<CtedObj>().deps().find(|s| s.id() == foreign.atom.owner))
.ok_or_else(|| foreign.clone())?
.get_card()
};
@@ -133,18 +129,80 @@ where A: AtomicFeatures {
Ok(TypAtom { value, data: foreign })
}
// #[derive(Clone)]
// pub struct SysCtx {
// pub reqnot: ReqNot<api::ExtMsgSet>,
// pub spawner: Spawner,
// pub id: api::SysId,
// pub cted: CtedObj,
// pub logger: Logger,
// pub obj_store: ObjStore,
// pub i: Rc<Interner>,
// }
// impl fmt::Debug for SysCtx {
// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// write!(f, "SysCtx({:?})", self.id)
// }
// }
#[derive(Clone)]
pub struct SysCtx {
pub reqnot: ReqNot<api::ExtMsgSet>,
pub spawner: Spawner,
pub id: api::SysId,
pub cted: CtedObj,
pub logger: Logger,
pub obj_store: ObjStore,
pub i: Rc<Interner>,
pub struct SysCtx(Rc<MemoMap<TypeId, Box<dyn Any>>>);
impl SysCtx {
pub fn new(
id: api::SysId,
i: Interner,
reqnot: ReqNot<ExtMsgSet>,
spawner: Spawner,
logger: Logger,
cted: CtedObj,
) -> Self {
let this = Self(Rc::new(MemoMap::new()));
this.add(id).add(i).add(reqnot).add(spawner).add(logger).add(cted);
this
}
pub fn add<T: SysCtxEntry>(&self, t: T) -> &Self {
assert!(self.0.insert(TypeId::of::<T>(), Box::new(t)), "Key already exists");
self
}
pub fn get_or_insert<T: SysCtxEntry>(&self, f: impl FnOnce() -> T) -> &T {
(self.0.get_or_insert_owned(TypeId::of::<T>(), || Box::new(f())).downcast_ref())
.expect("Keyed by TypeId")
}
pub fn get_or_default<T: SysCtxEntry + Default>(&self) -> &T {
self.get_or_insert(|| {
let rc_id = self.0.as_ref() as *const _ as *const () as usize;
eprintln!("Default-initializing {} in {}", type_name::<T>(), rc_id);
T::default()
})
}
pub fn try_get<T: SysCtxEntry>(&self) -> Option<&T> {
Some(self.0.get(&TypeId::of::<T>())?.downcast_ref().expect("Keyed by TypeId"))
}
pub fn get<T: SysCtxEntry>(&self) -> &T {
self.try_get().unwrap_or_else(|| panic!("Context {} missing", type_name::<T>()))
}
/// Shorthand to get the [Interner] instance
pub fn i(&self) -> &Interner { self.get::<Interner>() }
/// Shorthand to get the messaging link
pub fn reqnot(&self) -> &ReqNot<ExtMsgSet> { self.get::<ReqNot<ExtMsgSet>>() }
/// Shorthand to get the system ID
pub fn sys_id(&self) -> api::SysId { *self.get::<api::SysId>() }
/// Shorthand to get the task spawner callback
pub fn spawner(&self) -> &Spawner { self.get::<Spawner>() }
/// Shorthand to get the logger
pub fn logger(&self) -> &Logger { self.get::<Logger>() }
/// Shorthand to get the constructed system object
pub fn cted(&self) -> &CtedObj { self.get::<CtedObj>() }
}
impl fmt::Debug for SysCtx {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "SysCtx({:?})", self.id)
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "SysCtx({:?})", self.sys_id())
}
}
pub trait SysCtxEntry: 'static + Sized {}
impl SysCtxEntry for api::SysId {}
impl SysCtxEntry for ReqNot<api::ExtMsgSet> {}
impl SysCtxEntry for Spawner {}
impl SysCtxEntry for CtedObj {}
impl SysCtxEntry for Logger {}
impl SysCtxEntry for Interner {}

2
orchid-extension/src/system_ctor.rs
View File

@@ -90,7 +90,7 @@ impl<T: SystemCtor> DynSystemCtor for T {
mod dep_set_tuple_impls {
use orchid_base::box_chain;
use orchid_base::boxed_iter::BoxedIter;
use paste::paste;
use pastey::paste;
use super::{DepDef, DepSat};
use crate::api;

51
orchid-extension/src/tokio.rs Normal file
View File

@@ -0,0 +1,51 @@
use crate::entrypoint::ExtensionData;
#[cfg(feature = "tokio")]
pub async fn tokio_main(data: ExtensionData) {
use std::io::Write;
use std::mem;
use std::pin::Pin;
use std::rc::Rc;
use async_std::io;
use futures::StreamExt;
use futures::future::LocalBoxFuture;
use futures::stream::FuturesUnordered;
use orchid_api_traits::{Decode, Encode};
use tokio::task::{LocalSet, spawn_local};
use crate::api;
use crate::entrypoint::extension_init;
use crate::msg::{recv_parent_msg, send_parent_msg};
let local_set = LocalSet::new();
local_set.spawn_local(async {
let host_header = api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await;
let init =
Rc::new(extension_init(data, host_header, Rc::new(|fut| mem::drop(spawn_local(fut)))));
let mut buf = Vec::new();
init.header.encode(Pin::new(&mut buf)).await;
std::io::stdout().write_all(&buf).unwrap();
std::io::stdout().flush().unwrap();
// These are concurrent processes that never exit, so if the FuturesUnordered
// produces any result the extension should exit
let mut io = FuturesUnordered::<LocalBoxFuture<()>>::new();
io.push(Box::pin(async {
loop {
match recv_parent_msg().await {
Ok(msg) => init.send(&msg[..]).await,
Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break,
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break,
Err(e) => panic!("{e}"),
}
}
}));
io.push(Box::pin(async {
while let Some(msg) = init.recv().await {
send_parent_msg(&msg[..]).await.unwrap();
}
}));
io.next().await;
});
local_set.await;
}

249
orchid-extension/src/tree.rs
View File

@@ -1,99 +1,96 @@
use std::future::Future;
use std::num::NonZero;
use std::ops::Range;
use std::rc::Rc;
use async_stream::stream;
use dyn_clone::{DynClone, clone_box};
use futures::FutureExt;
use futures::future::{LocalBoxFuture, join_all};
use futures::{FutureExt, StreamExt};
use hashbrown::HashMap;
use itertools::Itertools;
use orchid_base::interner::Tok;
use orchid_base::interner::{Interner, Tok};
use orchid_base::location::SrcRange;
use orchid_base::name::Sym;
use orchid_base::reqnot::ReqHandlish;
use orchid_base::tree::{TokTree, Token};
use ordered_float::NotNan;
use orchid_base::tree::{TokTree, Token, TokenVariant};
use substack::Substack;
use trait_set::trait_set;
use crate::api;
use crate::atom::{AtomFactory, ForeignAtom};
use crate::conv::ToExpr;
use crate::entrypoint::MemberRecord;
use crate::expr::{Expr, ExprHandle};
use crate::func_atom::{ExprFunc, Fun};
use crate::gen_expr::{GExpr, arg, call, lambda, seq};
use crate::macros::Rule;
use crate::gen_expr::{GExpr, arg, call, lambda, seq, sym_ref};
use crate::system::SysCtx;
pub type GenTokTree<'a> = TokTree<'a, ForeignAtom<'a>, AtomFactory>;
pub type GenTok<'a> = Token<'a, ForeignAtom<'a>, AtomFactory>;
pub type GenTokTree = TokTree<Expr, GExpr>;
pub type GenTok = Token<Expr, GExpr>;
pub async fn do_extra(f: &AtomFactory, r: Range<u32>, ctx: SysCtx) -> api::TokenTree {
api::TokenTree { range: r, token: api::Token::Atom(f.clone().build(ctx).await) }
}
fn with_export(mem: GenMember, public: bool) -> Vec<GenItem> {
(public.then(|| GenItemKind::Export(mem.name.clone())).into_iter())
.chain([GenItemKind::Member(mem)])
.map(|kind| GenItem { comments: vec![], kind })
.collect()
}
pub struct GenItem {
pub kind: GenItemKind,
pub comments: Vec<String>,
}
impl GenItem {
pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Item {
let kind = match self.kind {
GenItemKind::Export(n) => api::ItemKind::Export(ctx.sys().i.i::<String>(&n).await.to_api()),
GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx).await),
GenItemKind::Import(cn) => api::ItemKind::Import(cn.tok().to_api()),
GenItemKind::Macro(priority, gen_rules) => {
let mut rules = Vec::with_capacity(gen_rules.len());
for rule in gen_rules {
rules.push(rule.into_api(ctx).await)
}
api::ItemKind::Macro(api::MacroBlock { priority, rules })
},
};
let comments = join_all(self.comments.iter().map(|c| async {
api::Comment {
location: api::Location::Inherit,
text: ctx.sys().i.i::<String>(c).await.to_api(),
}
}))
.await;
api::Item { location: api::Location::Inherit, comments, kind }
impl TokenVariant<api::Expression> for GExpr {
type FromApiCtx<'a> = ();
type ToApiCtx<'a> = (SysCtx, &'a dyn ReqHandlish);
async fn from_api(
_: &api::Expression,
_: &mut Self::FromApiCtx<'_>,
_: SrcRange,
_: &Interner,
) -> Self {
panic!("Received new expression from host")
}
async fn into_api(self, (ctx, hand): &mut Self::ToApiCtx<'_>) -> api::Expression {
self.api_return(ctx.clone(), hand).await
}
}
pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenItem> {
with_export(GenMember { name: name.to_string(), kind: MemKind::Const(value.to_expr()) }, public)
impl TokenVariant<api::ExprTicket> for Expr {
type FromApiCtx<'a> = SysCtx;
async fn from_api(
api: &api::ExprTicket,
ctx: &mut Self::FromApiCtx<'_>,
_: SrcRange,
_: &Interner,
) -> Self {
// SAFETY: receiving trees from sublexers implies ownership transfer
Expr::from_handle(Rc::new(unsafe { ExprHandle::from_args(ctx.clone(), *api) }))
}
type ToApiCtx<'a> = ();
async fn into_api(self, (): &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
let hand = self.handle();
std::mem::drop(self);
let h = match Rc::try_unwrap(hand) {
Ok(h) => h,
Err(h) => h.as_ref().clone().await,
};
h.into_tk()
}
}
pub fn x_tok(x: impl ToExpr) -> GenTok { GenTok::NewExpr(x.to_expr()) }
pub fn ref_tok(path: Sym) -> GenTok { GenTok::NewExpr(sym_ref(path)) }
pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenMember> {
vec![GenMember {
name: name.to_string(),
kind: MemKind::Const(value.to_expr()),
comments: vec![],
public,
}]
}
pub fn module(
public: bool,
name: &str,
imports: impl IntoIterator<Item = Sym>,
items: impl IntoIterator<Item = Vec<GenItem>>,
) -> Vec<GenItem> {
let (name, kind) = root_mod(name, imports, items);
with_export(GenMember { name, kind }, public)
mems: impl IntoIterator<Item = Vec<GenMember>>,
) -> Vec<GenMember> {
let (name, kind) = root_mod(name, mems);
vec![GenMember { name, kind, public, comments: vec![] }]
}
pub fn root_mod(
name: &str,
imports: impl IntoIterator<Item = Sym>,
items: impl IntoIterator<Item = Vec<GenItem>>,
) -> (String, MemKind) {
let kind = MemKind::Mod {
imports: imports.into_iter().collect(),
items: items.into_iter().flatten().collect(),
};
pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (String, MemKind) {
let kind = MemKind::Mod { members: mems.into_iter().flatten().collect() };
(name.to_string(), kind)
}
pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenItem> {
let fac = LazyMemberFactory::new(move |sym| async {
return MemKind::Const(build_lambdas(Fun::new(sym, xf).await, 0));
pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
let fac = LazyMemberFactory::new(move |sym, ctx| async {
return MemKind::Const(build_lambdas(Fun::new(sym, ctx, xf).await, 0));
fn build_lambdas(fun: Fun, i: u64) -> GExpr {
if i < fun.arity().into() {
return lambda(i, [build_lambdas(fun, i + 1)]);
@@ -106,20 +103,20 @@ pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<Gen
)
}
});
with_export(GenMember { name: name.to_string(), kind: MemKind::Lazy(fac) }, exported)
vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
}
pub fn macro_block(prio: Option<f64>, rules: impl IntoIterator<Item = Rule>) -> Vec<GenItem> {
let prio = prio.map(|p| NotNan::new(p).unwrap());
vec![GenItem {
kind: GenItemKind::Macro(prio, rules.into_iter().collect_vec()),
comments: vec![],
}]
pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
let mut items = items.into_iter().flatten().collect_vec();
for step in path.split("::").collect_vec().into_iter().rev() {
items = module(true, step, [items]);
}
items
}
pub fn comments<'a>(
cmts: impl IntoIterator<Item = &'a str>,
mut val: Vec<GenItem>,
) -> Vec<GenItem> {
mut val: Vec<GenMember>,
) -> Vec<GenMember> {
let cmts = cmts.into_iter().map(|c| c.to_string()).collect_vec();
for v in val.iter_mut() {
v.comments.extend(cmts.iter().cloned());
@@ -127,47 +124,77 @@ pub fn comments<'a>(
val
}
/// Trivially merge a gen tree. Behaviours were chosen to make this simple.
///
/// - Comments on imports are discarded
/// - Comments on exports and submodules are combined
/// - Duplicate constants result in an error
/// - A combination of lazy and anything results in an error
pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
let mut all_members = HashMap::<String, (MemKind, Vec<String>)>::new();
for mem in trees.into_iter().flatten() {
assert!(mem.public, "Non-trivial merge in {}", mem.name);
match mem.kind {
unit @ (MemKind::Const(_) | MemKind::Lazy(_)) => {
let prev = all_members.insert(mem.name.clone(), (unit, mem.comments.into_iter().collect()));
assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name);
},
MemKind::Mod { members } => match all_members.entry(mem.name.clone()) {
hashbrown::hash_map::Entry::Vacant(slot) => {
slot.insert((MemKind::Mod { members }, mem.comments.into_iter().collect()));
},
hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() {
(MemKind::Mod { members: old_items, .. }, old_cmts) => {
let mut swap = vec![];
std::mem::swap(&mut swap, old_items);
*old_items = merge_trivial([swap, members]);
old_cmts.extend(mem.comments);
},
_ => panic!("non-trivial merge on {}", mem.name),
},
},
}
}
(all_members.into_iter())
.map(|(name, (kind, comments))| GenMember { comments, kind, name, public: true })
.collect_vec()
}
trait_set! {
trait LazyMemberCallback =
FnOnce(Sym) -> LocalBoxFuture<'static, MemKind> + DynClone
FnOnce(Sym, SysCtx) -> LocalBoxFuture<'static, MemKind> + DynClone
}
pub struct LazyMemberFactory(Box<dyn LazyMemberCallback>);
impl LazyMemberFactory {
pub fn new<F: Future<Output = MemKind> + 'static>(
cb: impl FnOnce(Sym) -> F + Clone + 'static,
) -> Self {
Self(Box::new(|s| cb(s).boxed_local()))
pub fn new(cb: impl AsyncFnOnce(Sym, SysCtx) -> MemKind + Clone + 'static) -> Self {
Self(Box::new(|s, ctx| cb(s, ctx).boxed_local()))
}
pub async fn build(self, path: Sym) -> MemKind { (self.0)(path).await }
pub async fn build(self, path: Sym, ctx: SysCtx) -> MemKind { (self.0)(path, ctx).await }
}
impl Clone for LazyMemberFactory {
fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
}
pub enum GenItemKind {
Member(GenMember),
Export(String),
Import(Sym),
Macro(Option<NotNan<f64>>, Vec<Rule>),
}
pub struct GenMember {
name: String,
kind: MemKind,
pub name: String,
pub kind: MemKind,
pub public: bool,
pub comments: Vec<String>,
}
impl GenMember {
pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member {
let name = ctx.sys().i.i::<String>(&self.name).await;
api::Member {
kind: self.kind.into_api(&mut ctx.push_path(name.clone())).await,
name: name.to_api(),
}
let name = ctx.sys().i().i::<String>(&self.name).await;
let kind = self.kind.into_api(&mut ctx.push_path(name.clone())).await;
let comments =
join_all(self.comments.iter().map(|cmt| async { ctx.sys().i().i(cmt).await.to_api() })).await;
api::Member { kind, name: name.to_api(), comments, exported: self.public }
}
}
pub enum MemKind {
Const(GExpr),
Mod { imports: Vec<Sym>, items: Vec<GenItem> },
Mod { members: Vec<GenMember> },
Lazy(LazyMemberFactory),
}
impl MemKind {
@@ -175,16 +202,10 @@ impl MemKind {
match self {
Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await),
Self::Mod { imports, items } => {
let all_items = (imports.into_iter())
.map(|t| GenItem { comments: vec![], kind: GenItemKind::Import(t) })
.chain(items);
let mut items = Vec::new();
for i in all_items {
items.push(i.into_api(ctx).boxed_local().await)
}
api::MemberKind::Module(api::Module { items })
},
Self::Mod { members } => api::MemberKind::Module(api::Module {
members: Box::pin(stream! { for m in members { yield m.into_api(ctx).await } }.collect())
.await,
}),
}
}
}
@@ -192,27 +213,24 @@ impl MemKind {
pub trait TreeIntoApiCtx {
fn sys(&self) -> SysCtx;
fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
fn with_rule(&mut self, rule: Rc<Rule>) -> api::MacroId;
fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
fn req(&self) -> &impl ReqHandlish;
}
pub struct TIACtxImpl<'a, 'b, RH: ReqHandlish> {
pub struct TreeIntoApiCtxImpl<'a, 'b, RH: ReqHandlish> {
pub sys: SysCtx,
pub basepath: &'a [Tok<String>],
pub path: Substack<'a, Tok<String>>,
pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
pub rules: &'b mut HashMap<api::MacroId, Rc<Rule>>,
pub req: &'a RH,
}
impl<RH: ReqHandlish> TreeIntoApiCtx for TIACtxImpl<'_, '_, RH> {
impl<RH: ReqHandlish> TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_, RH> {
fn sys(&self) -> SysCtx { self.sys.clone() }
fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
TIACtxImpl {
TreeIntoApiCtxImpl {
req: self.req,
lazy_members: self.lazy_members,
rules: self.rules,
sys: self.sys.clone(),
basepath: self.basepath,
path: self.path.push(seg),
@@ -224,10 +242,5 @@ impl<RH: ReqHandlish> TreeIntoApiCtx for TIACtxImpl<'_, '_, RH> {
self.lazy_members.insert(id, MemberRecord::Gen(path, fac));
id
}
fn with_rule(&mut self, rule: Rc<Rule>) -> orchid_api::MacroId {
let id = api::MacroId(NonZero::new((self.lazy_members.len() + 1) as u64).unwrap());
self.rules.insert(id, rule);
id
}
fn req(&self) -> &impl ReqHandlish { self.req }
}

5
orchid-host/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "orchid-host"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -16,12 +16,13 @@ futures = "0.3.31"
hashbrown = "0.15.2"
itertools = "0.14.0"
lazy_static = "1.5.0"
memo-map = "0.3.3"
never = "0.1.0"
num-traits = "0.2.19"
orchid-api = { version = "0.1.0", path = "../orchid-api" }
orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
orchid-base = { version = "0.1.0", path = "../orchid-base" }
ordered-float = "4.6.0"
ordered-float = "5.0.0"
paste = "1.0.15"
substack = "1.1.1"
test_executors = "0.3.2"

10
orchid-host/src/atom.rs
View File

@@ -20,10 +20,12 @@ pub struct AtomData {
data: Vec<u8>,
}
impl AtomData {
#[must_use]
fn api(self) -> api::Atom {
let (owner, drop, data) = self.destructure();
api::Atom { data, drop, owner: owner.id() }
}
#[must_use]
fn api_ref(&self) -> api::Atom {
api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
}
@@ -48,6 +50,7 @@ impl fmt::Debug for AtomData {
#[derive(Clone, Debug)]
pub struct AtomHand(Rc<AtomData>);
impl AtomHand {
#[must_use]
pub(crate) async fn new(api::Atom { data, drop, owner }: api::Atom, ctx: &Ctx) -> Self {
let create = || async {
let owner = ctx.system_inst(owner).await.expect("Dropped system created atom");
@@ -67,6 +70,7 @@ impl AtomHand {
create().await
}
}
#[must_use]
pub async fn call(self, arg: Expr) -> api::Expression {
let owner_sys = self.0.owner.clone();
let reqnot = owner_sys.reqnot();
@@ -76,13 +80,18 @@ impl AtomHand {
Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
}
}
#[must_use]
pub fn sys(&self) -> &System { &self.0.owner }
#[must_use]
pub fn ext(&self) -> &Extension { self.sys().ext() }
pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
}
#[must_use]
pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
#[must_use]
pub async fn to_string(&self) -> String { take_first_fmt(self, &self.0.owner.ctx().i).await }
#[must_use]
pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) }
}
impl Format for AtomHand {
@@ -100,5 +109,6 @@ impl AtomRepr for AtomHand {
pub struct WeakAtomHand(Weak<AtomData>);
impl WeakAtomHand {
#[must_use]
pub fn upgrade(&self) -> Option<AtomHand> { self.0.upgrade().map(AtomHand) }
}

28
orchid-host/src/ctx.rs
View File

@@ -1,6 +1,6 @@
use std::cell::RefCell;
use std::num::{NonZero, NonZeroU16};
use std::rc::Rc;
use std::rc::{Rc, Weak};
use std::{fmt, ops};
use async_std::sync::RwLock;
@@ -11,16 +11,18 @@ use orchid_base::interner::Interner;
use crate::api;
use crate::atom::WeakAtomHand;
use crate::expr_store::ExprStore;
use crate::system::{System, WeakSystem};
use crate::tree::Module;
use crate::tree::WeakRoot;
pub struct CtxData {
pub i: Rc<Interner>,
pub i: Interner,
pub spawn: Spawner,
pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
pub system_id: RefCell<NonZeroU16>,
pub owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
pub root: RwLock<Module>,
pub common_exprs: ExprStore,
pub root: RwLock<WeakRoot>,
}
#[derive(Clone)]
pub struct Ctx(Rc<CtxData>);
@@ -28,25 +30,39 @@ impl ops::Deref for Ctx {
type Target = CtxData;
fn deref(&self) -> &Self::Target { &self.0 }
}
#[derive(Clone)]
pub struct WeakCtx(Weak<CtxData>);
impl WeakCtx {
#[must_use]
pub fn try_upgrade(&self) -> Option<Ctx> { Some(Ctx(self.0.upgrade()?)) }
#[must_use]
pub fn upgrade(&self) -> Ctx { self.try_upgrade().expect("Ctx manually kept alive until exit") }
}
impl Ctx {
#[must_use]
pub fn new(spawn: Spawner) -> Self {
Self(Rc::new(CtxData {
spawn,
i: Rc::default(),
i: Interner::default(),
systems: RwLock::default(),
system_id: RefCell::new(NonZero::new(1).unwrap()),
owned_atoms: RwLock::default(),
root: RwLock::new(Module::default()),
common_exprs: ExprStore::default(),
root: RwLock::default(),
}))
}
#[must_use]
pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {
self.systems.read().await.get(&id).and_then(WeakSystem::upgrade)
}
#[must_use]
pub(crate) fn next_sys_id(&self) -> api::SysId {
let mut g = self.system_id.borrow_mut();
*g = g.checked_add(1).unwrap_or(NonZeroU16::new(1).unwrap());
SysId(*g)
}
#[must_use]
pub fn downgrade(&self) -> WeakCtx { WeakCtx(Rc::downgrade(&self.0)) }
}
impl fmt::Debug for Ctx {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

259
orchid-host/src/dealias.rs
View File

@@ -1,16 +1,9 @@
use std::rc::Rc;
use futures::FutureExt;
use hashbrown::{HashMap, HashSet};
use itertools::{Either, Itertools};
use orchid_base::error::{OrcErr, Reporter, mk_err};
use orchid_base::format::{FmtCtxImpl, Format, take_first};
use hashbrown::HashSet;
use itertools::Itertools;
use orchid_base::error::{OrcErr, OrcRes, Reporter, mk_err, mk_errv};
use orchid_base::interner::{Interner, Tok};
use orchid_base::location::Pos;
use orchid_base::name::{NameLike, Sym, VName};
use crate::macros::{MacTok, MacTree};
use crate::tree::{ItemKind, MemberKind, Module, RuleKind, WalkErrorKind};
use orchid_base::name::VName;
/// Errors produced by absolute_path
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -48,193 +41,107 @@ impl AbsPathError {
///
/// if the relative path contains as many or more `super` segments than the
/// length of the absolute path.
pub fn absolute_path(
pub async fn absolute_path(
mut cwd: &[Tok<String>],
mut rel: &[Tok<String>],
i: &Interner,
) -> Result<VName, AbsPathError> {
let mut relative = false;
if rel.first().map(|t| t.as_str()) == Some("self") {
relative = true;
rel = rel.split_first().expect("checked above").1;
let i_self = i.i("self").await;
let i_super = i.i("super").await;
let relative = rel.first().is_some_and(|s| *s != i_self && *s != i_super);
if let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h != i_self) {
rel = tail;
} else {
while rel.first().map(|t| t.as_str()) == Some("super") {
match cwd.split_last() {
Some((_, torso)) => cwd = torso,
None => return Err(AbsPathError::TooManySupers),
};
rel = rel.split_first().expect("checked above").1;
relative = true;
while let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h == i_super) {
cwd = cwd.split_last().ok_or(AbsPathError::TooManySupers)?.1;
rel = tail;
}
}
match relative {
true => VName::new(cwd.iter().chain(rel).cloned()),
false => VName::new(rel.to_vec()),
}
.map_err(|_| AbsPathError::RootPath)
if relative { VName::new(cwd.iter().chain(rel).cloned()) } else { VName::new(rel.to_vec()) }
.map_err(|_| AbsPathError::RootPath)
}
pub async fn resolv_glob(
pub struct DealiasCtx<'a> {
pub i: &'a Interner,
pub rep: &'a Reporter,
}
pub async fn resolv_glob<Mod: Tree>(
cwd: &[Tok<String>],
root: &Module,
root: &Mod,
abs_path: &[Tok<String>],
pos: Pos,
i: &Interner,
r: &impl Reporter,
) -> Vec<Tok<String>> {
ctx: &mut Mod::Ctx<'_>,
) -> OrcRes<HashSet<Tok<String>>> {
let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
let (co_prefix, diff_path) = abs_path.split_at(coprefix_len);
let co_parent = root.walk(false, co_prefix.iter().cloned()).await.expect("Invalid step in cwd");
let target_module = match co_parent.walk(true, diff_path.iter().cloned()).await {
let (co_prefix, diff_path) = abs_path.split_at(abs_path.len().min(coprefix_len + 1));
let fst_diff =
walk(root, false, co_prefix.iter().cloned(), ctx).await.expect("Invalid step in cwd");
let target_module = match walk(fst_diff, true, diff_path.iter().cloned(), ctx).await {
Ok(t) => t,
Err(e) => {
let path = abs_path[..=coprefix_len + e.pos].iter().join("::");
let (tk, msg) = match e.kind {
WalkErrorKind::Constant =>
(i.i("Invalid import path").await, format!("{path} is a constant")),
WalkErrorKind::Missing => (i.i("Invalid import path").await, format!("{path} not found")),
WalkErrorKind::Private => (i.i("Import inaccessible").await, format!("{path} is private")),
ChildErrorKind::Constant => ("Invalid import path", format!("{path} is a const")),
ChildErrorKind::Missing => ("Invalid import path", format!("{path} not found")),
ChildErrorKind::Private => ("Import inaccessible", format!("{path} is private")),
};
r.report(mk_err(tk, msg, [pos.into()]));
return vec![];
return Err(mk_errv(i.i(tk).await, msg, [pos]));
},
};
target_module.exports.clone()
Ok(target_module.children(coprefix_len < abs_path.len()))
}
/// Read import statements and convert them into aliases, rasising any import
/// errors in the process
pub async fn imports_to_aliases(
module: &Module,
cwd: &mut Vec<Tok<String>>,
root: &Module,
alias_map: &mut HashMap<Sym, Sym>,
alias_rev_map: &mut HashMap<Sym, HashSet<Sym>>,
i: &Interner,
rep: &impl Reporter,
) {
let mut import_locs = HashMap::<Sym, Vec<Pos>>::new();
for item in &module.items {
match &item.kind {
ItemKind::Import(imp) => match absolute_path(cwd, &imp.path) {
Err(e) => rep.report(e.err_obj(i, item.pos.clone(), &imp.path.iter().join("::")).await),
Ok(abs_path) => {
let names = match imp.name.as_ref() {
Some(n) => Either::Right([n.clone()].into_iter()),
None => Either::Left(
resolv_glob(cwd, root, &abs_path, item.pos.clone(), i, rep).await.into_iter(),
),
};
for name in names {
let mut tgt = abs_path.clone().suffix([name.clone()]).to_sym(i).await;
let src = Sym::new(cwd.iter().cloned().chain([name]), i).await.unwrap();
import_locs.entry(src.clone()).or_insert(vec![]).push(item.pos.clone());
if let Some(tgt2) = alias_map.get(&tgt) {
tgt = tgt2.clone();
}
if src == tgt {
rep.report(mk_err(
i.i("Circular references").await,
format!("{src} circularly refers to itself"),
[item.pos.clone().into()],
));
continue;
}
if let Some(fst_val) = alias_map.get(&src) {
let locations = (import_locs.get(&src))
.expect("The same name could only have appeared in the same module");
rep.report(mk_err(
i.i("Conflicting imports").await,
if fst_val == &src {
format!("{src} is imported multiple times")
} else {
format!("{} could refer to both {fst_val} and {src}", src.last())
},
locations.iter().map(|p| p.clone().into()).collect_vec(),
))
}
let mut srcv = vec![src.clone()];
if let Some(src_extra) = alias_rev_map.remove(&src) {
srcv.extend(src_extra);
}
for src in srcv {
alias_map.insert(src.clone(), tgt.clone());
alias_rev_map.entry(tgt.clone()).or_insert(HashSet::new()).insert(src);
}
}
},
},
ItemKind::Member(mem) => match mem.kind().await {
MemberKind::Const(_) => (),
MemberKind::Mod(m) => {
cwd.push(mem.name());
imports_to_aliases(m, cwd, root, alias_map, alias_rev_map, i, rep).boxed_local().await;
cwd.pop();
},
},
ItemKind::Export(_) | ItemKind::Macro(..) => (),
pub type ChildResult<'a, T> = Result<&'a T, ChildErrorKind>;
pub trait Tree {
type Ctx<'a>;
#[must_use]
fn children(&self, public_only: bool) -> HashSet<Tok<String>>;
#[must_use]
fn child(
&self,
key: Tok<String>,
public_only: bool,
ctx: &mut Self::Ctx<'_>,
) -> impl Future<Output = ChildResult<'_, Self>>;
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub enum ChildErrorKind {
Missing,
/// Only thrown if public_only is true
Private,
Constant,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct ChildError {
pub pos: usize,
pub kind: ChildErrorKind,
}
// Problem: walk should take into account aliases and visibility
//
// help: since every alias is also its own import, visibility only has to be
// checked on the top level
//
// idea: do a simple stack machine like below with no visibility for aliases and
// call it from an access-checking implementation for just the top level
//
// caveat: we need to check EVERY IMPORT to ensure that all
// errors are raised
pub async fn walk<'a, T: Tree>(
root: &'a T,
public_only: bool,
path: impl IntoIterator<Item = Tok<String>>,
ctx: &mut T::Ctx<'_>,
) -> Result<&'a T, ChildError> {
let mut cur = root;
for (i, item) in path.into_iter().enumerate() {
match cur.child(item, public_only, ctx).await {
ChildResult::Ok(v) => cur = v,
ChildResult::Err(kind) => return Err(ChildError { pos: i, kind }),
}
}
}
pub async fn dealias(module: &mut Module, alias_map: &HashMap<Sym, Sym>, i: &Interner) {
for item in &mut module.items {
match &mut item.kind {
ItemKind::Export(_) | ItemKind::Import(_) => (),
ItemKind::Member(mem) => match mem.kind_mut().await {
MemberKind::Const(c) => {
let Some(source) = c.source() else { continue };
let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
c.set_source(new_source);
},
MemberKind::Mod(m) => dealias(m, alias_map, i).boxed_local().await,
},
ItemKind::Macro(_, rules) =>
for rule in rules.iter_mut() {
let RuleKind::Native(c) = &mut rule.kind else { continue };
let Some(source) = c.source() else { continue };
let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
c.set_source(new_source);
},
}
}
}
async fn dealias_mactree(
mtree: &MacTree,
aliases: &HashMap<Sym, Sym>,
i: &Interner,
) -> Option<MacTree> {
let new_tok = match &*mtree.tok {
MacTok::Atom(_) | MacTok::Ph(_) => return None,
tok @ (MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_)) => panic!(
"{} should not appear in retained pre-macro source",
take_first(&tok.print(&FmtCtxImpl { i }).await, true)
),
MacTok::Name(n) => MacTok::Name(aliases.get(n).unwrap_or(n).clone()),
MacTok::Lambda(arg, body) => {
match (dealias_mactreev(arg, aliases, i).await, dealias_mactreev(body, aliases, i).await) {
(None, None) => return None,
(Some(arg), None) => MacTok::Lambda(arg, body.clone()),
(None, Some(body)) => MacTok::Lambda(arg.clone(), body),
(Some(arg), Some(body)) => MacTok::Lambda(arg, body),
}
},
MacTok::S(p, b) => MacTok::S(*p, dealias_mactreev(b, aliases, i).await?),
};
Some(MacTree { pos: mtree.pos.clone(), tok: Rc::new(new_tok) })
}
async fn dealias_mactreev(
mtreev: &[MacTree],
aliases: &HashMap<Sym, Sym>,
i: &Interner,
) -> Option<Vec<MacTree>> {
let mut results = Vec::with_capacity(mtreev.len());
let mut any_some = false;
for item in mtreev {
let out = dealias_mactree(item, aliases, i).boxed_local().await;
any_some |= out.is_some();
results.push(out.unwrap_or(item.clone()));
}
any_some.then_some(results)
Ok(cur)
}

72
orchid-host/src/execute.rs
View File

@@ -3,15 +3,14 @@ use std::mem;
use async_std::sync::RwLockWriteGuard;
use bound::Bound;
use futures::FutureExt;
use orchid_base::error::{OrcErrv, mk_errv};
use orchid_base::error::OrcErrv;
use orchid_base::format::{FmtCtxImpl, Format, take_first};
use orchid_base::location::Pos;
use orchid_base::logging::Logger;
use orchid_base::name::NameLike;
use crate::ctx::Ctx;
use crate::expr::{Expr, ExprKind, PathSet, Step};
use crate::tree::{ItemKind, MemberKind};
use crate::expr::{Expr, ExprKind, ExprParseCtx, PathSet, PathSetBuilder, Step};
use crate::tree::Root;
type ExprGuard = Bound<RwLockWriteGuard<'static, ExprKind>, Expr>;
@@ -38,16 +37,21 @@ pub struct ExecCtx {
cur_pos: Pos,
did_pop: bool,
logger: Logger,
root: Root,
}
impl ExecCtx {
pub async fn new(ctx: Ctx, logger: Logger, init: Expr) -> Self {
#[must_use]
pub async fn new(ctx: Ctx, logger: Logger, root: Root, init: Expr) -> Self {
let cur_pos = init.pos();
let cur = Bound::async_new(init, |init| init.kind().write()).await;
Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger }
Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger, root }
}
#[must_use]
pub fn remaining_gas(&self) -> u64 { self.gas.expect("queried remaining_gas but no gas was set") }
pub fn set_gas(&mut self, gas: Option<u64>) { self.gas = gas }
#[must_use]
pub fn idle(&self) -> bool { self.did_pop }
#[must_use]
pub fn result(self) -> ExecResult {
if self.idle() {
match &*self.cur {
@@ -58,15 +62,18 @@ impl ExecCtx {
ExecResult::Gas(self)
}
}
#[must_use]
pub fn use_gas(&mut self, amount: u64) -> bool {
if let Some(gas) = &mut self.gas {
*gas -= amount;
}
self.gas != Some(0)
}
#[must_use]
pub async fn try_lock(&self, ex: &Expr) -> ExprGuard {
Bound::async_new(ex.clone(), |ex| ex.kind().write()).await
}
#[must_use]
pub async fn unpack_ident(&self, ex: &Expr) -> Expr {
match ex.kind().try_write().as_deref_mut() {
Some(ExprKind::Identity(ex)) => {
@@ -91,47 +98,22 @@ impl ExecCtx {
},
ExprKind::Seq(a, b) if !self.did_pop => (ExprKind::Seq(a.clone(), b), StackOp::Push(a)),
ExprKind::Seq(_, b) => (ExprKind::Identity(b), StackOp::Nop),
ExprKind::Const(name) => {
let (cn, mp) = name.split_last();
let root_lock = self.ctx.root.read().await;
let module = root_lock.walk(true, mp.iter().cloned()).await.unwrap();
let member = (module.items.iter())
.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
.find(|m| m.name() == cn);
match member {
None => (
ExprKind::Bottom(mk_errv(
self.ctx.i.i("Constant does not exist").await,
format!("{name} does not refer to a constant"),
[self.cur_pos.clone().into()],
)),
StackOp::Pop,
),
Some(mem) => match mem.kind().await {
MemberKind::Mod(_) => (
ExprKind::Bottom(mk_errv(
self.ctx.i.i("module used as constant").await,
format!("{name} is a module"),
[self.cur_pos.clone().into()],
)),
StackOp::Pop,
),
MemberKind::Const(c) => {
let value = c.get_bytecode(&self.ctx).await;
(ExprKind::Identity(value.clone()), StackOp::Nop)
},
},
}
},
ExprKind::Const(name) =>
match self.root.get_const_value(name, self.cur_pos.clone()).await {
Err(e) => (ExprKind::Bottom(e), StackOp::Pop),
Ok(v) => (ExprKind::Identity(v), StackOp::Nop),
},
ExprKind::Arg => panic!("This should not appear outside function bodies"),
ek @ ExprKind::Atom(_) => (ek, StackOp::Pop),
ExprKind::Bottom(bot) => (ExprKind::Bottom(bot.clone()), StackOp::Unwind(bot)),
ExprKind::Call(f, x) if !self.did_pop => (ExprKind::Call(f.clone(), x), StackOp::Push(f)),
ExprKind::Call(f, x) => match f.try_into_owned_atom().await {
Ok(atom) => {
let mut ext = atom.sys().ext().clone();
let ext = atom.sys().ext().clone();
let x_norm = self.unpack_ident(&x).await;
let val = Expr::from_api(&atom.call(x_norm).await, &mut ext).await;
let mut parse_ctx = ExprParseCtx { ctx: &self.ctx, exprs: ext.exprs() };
let val =
Expr::from_api(&atom.call(x_norm).await, PathSetBuilder::new(), &mut parse_ctx).await;
(ExprKind::Identity(val.clone()), StackOp::Swap(val))
},
Err(f) => match &*f.kind().read().await {
@@ -139,9 +121,14 @@ impl ExecCtx {
panic!("This should not appear outside function bodies"),
ExprKind::Missing => panic!("Should have been replaced"),
ExprKind::Atom(a) => {
let mut ext = a.sys().ext().clone();
let ext = a.sys().ext().clone();
let x_norm = self.unpack_ident(&x).await;
let val = Expr::from_api(&a.clone().call(x_norm).await, &mut ext).await;
let val = Expr::from_api(
&a.clone().call(x_norm).await,
PathSetBuilder::new(),
&mut ExprParseCtx { ctx: ext.ctx(), exprs: ext.exprs() },
)
.await;
(ExprKind::Identity(val.clone()), StackOp::Swap(val))
},
ExprKind::Bottom(exprv) => (ExprKind::Bottom(exprv.clone()), StackOp::Pop),
@@ -185,6 +172,7 @@ impl ExecCtx {
}
}
#[must_use]
async fn substitute(
src: &Expr,
path: &[Step],

363
orchid-host/src/expr.rs
View File

@@ -8,23 +8,25 @@ use async_std::sync::RwLock;
use futures::FutureExt;
use hashbrown::HashSet;
use itertools::Itertools;
use orchid_base::error::{OrcErrv, mk_errv};
use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, Format, Variants, take_first};
use orchid_base::location::Pos;
use orchid_base::macros::mtreev_fmt;
use orchid_base::error::OrcErrv;
use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
use orchid_base::interner::Interner;
use orchid_base::location::{Pos, SrcRange};
use orchid_base::name::Sym;
use orchid_base::tokens::Paren;
use orchid_base::tree::{AtomRepr, indent};
use orchid_base::{match_mapping, tl_cache};
use orchid_base::tl_cache;
use orchid_base::tree::{AtomRepr, TokenVariant, indent};
use substack::Substack;
use crate::api;
use crate::atom::AtomHand;
use crate::ctx::Ctx;
use crate::extension::Extension;
use crate::macros::{MacTok, MacTree};
use crate::expr_store::ExprStore;
pub type ExprParseCtx = Extension;
#[derive(Clone)]
pub struct ExprParseCtx<'a> {
pub ctx: &'a Ctx,
pub exprs: &'a ExprStore,
}
#[derive(Debug)]
pub struct ExprData {
@@ -35,6 +37,7 @@ pub struct ExprData {
#[derive(Clone, Debug)]
pub struct Expr(Rc<ExprData>);
impl Expr {
#[must_use]
pub fn pos(&self) -> Pos { self.0.pos.clone() }
pub async fn try_into_owned_atom(self) -> Result<AtomHand, Self> {
match Rc::try_unwrap(self.0) {
@@ -45,24 +48,60 @@ impl Expr {
},
}
}
#[must_use]
pub async fn as_atom(&self) -> Option<AtomHand> {
if let ExprKind::Atom(a) = &*self.kind().read().await { Some(a.clone()) } else { None }
}
#[must_use]
pub fn strong_count(&self) -> usize { Rc::strong_count(&self.0) }
#[must_use]
pub fn id(&self) -> api::ExprTicket {
api::ExprTicket(
NonZeroU64::new(self.0.as_ref() as *const ExprData as usize as u64)
.expect("this is a ref, it cannot be null"),
)
}
pub async fn from_api(api: &api::Expression, ctx: &mut ExprParseCtx) -> Self {
if let api::ExpressionKind::Slot(tk) = &api.kind {
return ctx.exprs().get_expr(*tk).expect("Invalid slot");
}
let pos = Pos::from_api(&api.location, &ctx.ctx().i).await;
let kind = RwLock::new(ExprKind::from_api(&api.kind, pos.clone(), ctx).boxed_local().await);
Self(Rc::new(ExprData { pos, kind }))
#[must_use]
pub async fn from_api(
api: &api::Expression,
psb: PathSetBuilder<'_, u64>,
ctx: &mut ExprParseCtx<'_>,
) -> Self {
let pos = Pos::from_api(&api.location, &ctx.ctx.i).await;
let kind = match &api.kind {
api::ExpressionKind::Arg(n) => {
assert!(psb.register_arg(n), "Arguments must be enclosed in a matching lambda");
ExprKind::Arg
},
api::ExpressionKind::Bottom(bot) =>
ExprKind::Bottom(OrcErrv::from_api(bot, &ctx.ctx.i).await),
api::ExpressionKind::Call(f, x) => {
let (lpsb, rpsb) = psb.split();
ExprKind::Call(
Expr::from_api(f, lpsb, ctx).boxed_local().await,
Expr::from_api(x, rpsb, ctx).boxed_local().await,
)
},
api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name, &ctx.ctx.i).await),
api::ExpressionKind::Lambda(x, body) => {
let lbuilder = psb.lambda(x);
let body = Expr::from_api(body, lbuilder.stack(), ctx).boxed_local().await;
ExprKind::Lambda(lbuilder.collect(), body)
},
api::ExpressionKind::NewAtom(a) =>
ExprKind::Atom(AtomHand::from_api(a, pos.clone(), &mut ctx.ctx.clone()).await),
api::ExpressionKind::Slot(tk) => return ctx.exprs.get_expr(*tk).expect("Invalid slot"),
api::ExpressionKind::Seq(a, b) => {
let (apsb, bpsb) = psb.split();
ExprKind::Seq(
Expr::from_api(a, apsb, ctx).boxed_local().await,
Expr::from_api(b, bpsb, ctx).boxed_local().await,
)
},
};
Self(Rc::new(ExprData { pos, kind: RwLock::new(kind) }))
}
#[must_use]
pub async fn to_api(&self) -> api::InspectedKind {
use api::InspectedKind as K;
match &*self.0.kind.read().await {
@@ -72,6 +111,7 @@ impl Expr {
_ => K::Opaque,
}
}
#[must_use]
pub fn kind(&self) -> &RwLock<ExprKind> { &self.0.kind }
}
impl Format for Expr {
@@ -107,23 +147,7 @@ pub enum ExprKind {
Missing,
}
impl ExprKind {
pub async fn from_api(api: &api::ExpressionKind, pos: Pos, ctx: &mut ExprParseCtx) -> Self {
match_mapping!(api, api::ExpressionKind => ExprKind {
Lambda(id => PathSet::from_api(*id, api), b => Expr::from_api(b, ctx).await),
Bottom(b => OrcErrv::from_api(b, &ctx.ctx().i).await),
Call(f => Expr::from_api(f, ctx).await, x => Expr::from_api(x, ctx).await),
Const(c => Sym::from_api(*c, &ctx.ctx().i).await),
Seq(a => Expr::from_api(a, ctx).await, b => Expr::from_api(b, ctx).await),
} {
api::ExpressionKind::Arg(_) => ExprKind::Arg,
api::ExpressionKind::NewAtom(a) => ExprKind::Atom(AtomHand::from_api(
a,
pos,
&mut ctx.ctx().clone()
).await),
api::ExpressionKind::Slot(_) => panic!("Handled in Expr"),
})
}
#[must_use]
pub fn at(self, pos: Pos) -> Expr { Expr(Rc::new(ExprData { pos, kind: RwLock::new(self) })) }
}
impl Format for ExprKind {
@@ -174,6 +198,102 @@ pub enum Step {
Right,
}
#[derive(Clone)]
pub enum PathSetFrame<'a, T: PartialEq> {
Lambda(&'a T, &'a RefCell<Option<PathSet>>),
Step(Step),
}
#[derive(Clone)]
pub struct PathSetBuilder<'a, T: PartialEq>(Substack<'a, PathSetFrame<'a, T>>);
impl<'a, T: PartialEq> PathSetBuilder<'a, T> {
#[must_use]
pub fn new() -> Self { Self(Substack::Bottom) }
#[must_use]
pub fn split(&'a self) -> (Self, Self) {
(
Self(self.0.push(PathSetFrame::Step(Step::Left))),
Self(self.0.push(PathSetFrame::Step(Step::Right))),
)
}
#[must_use]
pub fn lambda<'b>(self, arg: &'b T) -> LambdaBuilder<'b, T>
where 'a: 'b {
LambdaBuilder { arg, path: RefCell::default(), stack: self }
}
/// Register an argument with the corresponding lambda and return true if one
/// was found. (if false is returned, the name is unbound and may refer to a
/// global)
pub fn register_arg(self, t: &T) -> bool {
let mut steps = VecDeque::new();
for step in self.0.iter() {
match step {
PathSetFrame::Step(step) => steps.push_front(*step),
PathSetFrame::Lambda(name, _) if **name != *t => (),
PathSetFrame::Lambda(_, cell) => {
let mut ps_opt = cell.borrow_mut();
match &mut *ps_opt {
val @ None => *val = Some(PathSet { steps: steps.into(), next: None }),
Some(val) => {
let mut swap = PathSet { steps: Vec::new(), next: None };
mem::swap(&mut swap, val);
*val = merge(swap, &Vec::from(steps));
},
}
return true;
},
};
}
return false;
fn merge(ps: PathSet, steps: &[Step]) -> PathSet {
let diff_idx = ps.steps.iter().zip(steps).take_while(|(l, r)| l == r).count();
if diff_idx == ps.steps.len() {
if diff_idx == steps.len() {
match ps.next {
Some(_) => panic!("New path ends where old path forks"),
None => panic!("New path same as old path"),
}
}
let Some((left, right)) = ps.next else { panic!("Old path ends where new path continues") };
let next = match steps[diff_idx] {
Step::Left => Some((Box::new(merge(*left, &steps[diff_idx + 1..])), right)),
Step::Right => Some((left, Box::new(merge(*right, &steps[diff_idx + 1..])))),
};
PathSet { steps: ps.steps, next }
} else {
let shared_steps = ps.steps.iter().take(diff_idx).cloned().collect();
let main_steps = ps.steps.iter().skip(diff_idx + 1).cloned().collect();
let new_branch = steps[diff_idx + 1..].to_vec();
let main_side = PathSet { steps: main_steps, next: ps.next };
let new_side = PathSet { steps: new_branch, next: None };
let (left, right) = match steps[diff_idx] {
Step::Left => (new_side, main_side),
Step::Right => (main_side, new_side),
};
PathSet { steps: shared_steps, next: Some((Box::new(left), Box::new(right))) }
}
}
}
}
impl<'a, T: PartialEq> Default for PathSetBuilder<'a, T> {
fn default() -> Self { Self::new() }
}
pub struct LambdaBuilder<'a, T: PartialEq> {
arg: &'a T,
path: RefCell<Option<PathSet>>,
stack: PathSetBuilder<'a, T>,
}
impl<'a, T: PartialEq> LambdaBuilder<'a, T> {
#[must_use]
pub fn stack(&'a self) -> PathSetBuilder<'a, T> {
PathSetBuilder(self.stack.0.push(PathSetFrame::Lambda(self.arg, &self.path)))
}
#[must_use]
pub fn collect(self) -> Option<PathSet> { self.path.into_inner() }
}
#[derive(Clone, Debug)]
pub struct PathSet {
/// The single steps through [super::nort::Clause::Apply]
@@ -183,35 +303,10 @@ pub struct PathSet {
pub next: Option<(Box<PathSet>, Box<PathSet>)>,
}
impl PathSet {
#[must_use]
pub fn next(&self) -> Option<(&PathSet, &PathSet)> {
self.next.as_ref().map(|(l, r)| (&**l, &**r))
}
pub fn from_api(id: u64, api: &api::ExpressionKind) -> Option<Self> {
use api::ExpressionKind as K;
struct Suffix(VecDeque<Step>, Option<(Box<PathSet>, Box<PathSet>)>);
fn seal(Suffix(steps, next): Suffix) -> PathSet { PathSet { steps: steps.into(), next } }
fn after(step: Step, mut suf: Suffix) -> Suffix {
suf.0.push_front(step);
suf
}
return from_api_inner(id, api).map(seal);
fn from_api_inner(id: u64, api: &api::ExpressionKind) -> Option<Suffix> {
match &api {
K::Arg(id2) => (id == *id2).then_some(Suffix(VecDeque::new(), None)),
K::Bottom(_) | K::Const(_) | K::NewAtom(_) | K::Slot(_) => None,
K::Lambda(_, b) => from_api_inner(id, &b.kind),
K::Call(l, r) | K::Seq(l, r) => {
match (from_api_inner(id, &l.kind), from_api_inner(id, &r.kind)) {
(Some(a), Some(b)) =>
Some(Suffix(VecDeque::new(), Some((Box::new(seal(a)), Box::new(seal(b)))))),
(Some(l), None) => Some(after(Step::Left, l)),
(None, Some(r)) => Some(after(Step::Right, r)),
(None, None) => None,
}
},
}
}
}
}
impl fmt::Display for PathSet {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -229,6 +324,7 @@ impl fmt::Display for PathSet {
}
}
#[must_use]
pub fn bot_expr(err: impl Into<OrcErrv>) -> Expr {
let errv: OrcErrv = err.into();
let pos = errv.pos_iter().next().map_or(Pos::None, |ep| ep.position.clone());
@@ -237,123 +333,44 @@ pub fn bot_expr(err: impl Into<OrcErrv>) -> Expr {
pub struct WeakExpr(Weak<ExprData>);
impl WeakExpr {
#[must_use]
pub fn upgrade(&self) -> Option<Expr> { self.0.upgrade().map(Expr) }
}
#[derive(Clone)]
pub enum SrcToExprStep<'a> {
Left,
Right,
Lambda(Sym, &'a RefCell<Option<PathSet>>),
impl TokenVariant<api::ExprTicket> for Expr {
type FromApiCtx<'a> = ExprStore;
async fn from_api(
api: &api::ExprTicket,
ctx: &mut Self::FromApiCtx<'_>,
_: SrcRange,
_: &Interner,
) -> Self {
ctx.get_expr(*api).expect("Invalid ticket")
}
type ToApiCtx<'a> = ExprStore;
async fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
let id = self.id();
ctx.give_expr(self);
id
}
}
pub async fn mtreev_to_expr(
src: &[MacTree],
stack: Substack<'_, SrcToExprStep<'_>>,
ctx: &Ctx,
) -> ExprKind {
let Some((x, f)) = src.split_last() else { panic!("Empty expression cannot be evaluated") };
let x_stack = if f.is_empty() { stack.clone() } else { stack.push(SrcToExprStep::Right) };
let x_kind = match &*x.tok {
MacTok::Atom(a) => ExprKind::Atom(a.clone()),
MacTok::Name(n) => 'name: {
let mut steps = VecDeque::new();
for step in x_stack.iter() {
match step {
SrcToExprStep::Left => steps.push_front(Step::Left),
SrcToExprStep::Right => steps.push_front(Step::Right),
SrcToExprStep::Lambda(name, _) if name != n => continue,
SrcToExprStep::Lambda(_, cell) => {
let mut ps = cell.borrow_mut();
match &mut *ps {
val @ None => *val = Some(PathSet { steps: steps.into(), next: None }),
Some(val) => {
let mut swap = PathSet { steps: Vec::new(), next: None };
mem::swap(&mut swap, val);
*val = merge(swap, &Vec::from(steps));
fn merge(ps: PathSet, steps: &[Step]) -> PathSet {
let diff_idx = ps.steps.iter().zip(steps).take_while(|(l, r)| l == r).count();
if diff_idx == ps.steps.len() {
if diff_idx == steps.len() {
match ps.next {
Some(_) => panic!("New path ends where old path forks"),
None => panic!("New path same as old path"),
}
}
let Some((left, right)) = ps.next else {
panic!("Old path ends where new path continues")
};
let next = match steps[diff_idx] {
Step::Left => Some((Box::new(merge(*left, &steps[diff_idx + 1..])), right)),
Step::Right => Some((left, Box::new(merge(*right, &steps[diff_idx + 1..])))),
};
PathSet { steps: ps.steps, next }
} else {
let shared_steps = ps.steps.iter().take(diff_idx).cloned().collect();
let main_steps = ps.steps.iter().skip(diff_idx + 1).cloned().collect();
let new_branch = steps[diff_idx + 1..].to_vec();
let main_side = PathSet { steps: main_steps, next: ps.next };
let new_side = PathSet { steps: new_branch, next: None };
let (left, right) = match steps[diff_idx] {
Step::Left => (new_side, main_side),
Step::Right => (main_side, new_side),
};
PathSet { steps: shared_steps, next: Some((Box::new(left), Box::new(right))) }
}
}
},
}
break 'name ExprKind::Arg;
},
}
}
ExprKind::Const(n.clone())
},
MacTok::Ph(_) | MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_) =>
ExprKind::Bottom(mk_errv(
ctx.i.i("placeholder in value").await,
"Placeholders cannot appear anywhere outside macro patterns",
[x.pos.clone().into()],
)),
MacTok::S(Paren::Round, b) if b.is_empty() =>
return ExprKind::Bottom(mk_errv(
ctx.i.i("Empty expression").await,
"Empty parens () are illegal",
[x.pos.clone().into()],
)),
MacTok::S(Paren::Round, b) => mtreev_to_expr(b, x_stack, ctx).boxed_local().await,
MacTok::S(..) => ExprKind::Bottom(mk_errv(
ctx.i.i("non-round parentheses after macros").await,
"[] or {} block was not consumed by macros; expressions may only contain ()",
[x.pos.clone().into()],
)),
MacTok::Lambda(_, b) if b.is_empty() =>
return ExprKind::Bottom(mk_errv(
ctx.i.i("Empty lambda").await,
"Lambdas must have a body",
[x.pos.clone().into()],
)),
MacTok::Lambda(arg, b) => 'lambda_converter: {
if let [MacTree { tok, .. }] = &**arg {
if let MacTok::Name(n) = &**tok {
let path = RefCell::new(None);
let b = mtreev_to_expr(b, x_stack.push(SrcToExprStep::Lambda(n.clone(), &path)), ctx)
.boxed_local()
.await;
break 'lambda_converter ExprKind::Lambda(path.into_inner(), b.at(x.pos.clone()));
}
}
let argstr = take_first(&mtreev_fmt(arg, &FmtCtxImpl { i: &ctx.i }).await, true);
ExprKind::Bottom(mk_errv(
ctx.i.i("Malformeed lambda").await,
format!("Lambda argument should be single name, found {argstr}"),
[x.pos.clone().into()],
))
},
};
if f.is_empty() {
return x_kind;
/// Acknowledgment that expr serialization is impossible and thus will panic.
#[derive(Debug, Clone, Copy, Default)]
pub struct ExprWillPanic;
impl TokenVariant<api::Expression> for Expr {
type FromApiCtx<'a> = ExprParseCtx<'a>;
async fn from_api(
api: &api::Expression,
ctx: &mut Self::FromApiCtx<'_>,
_: SrcRange,
_: &Interner,
) -> Self {
Self::from_api(api, PathSetBuilder::new(), ctx).await
}
type ToApiCtx<'a> = ExprWillPanic;
async fn into_api(self, ExprWillPanic: &mut Self::ToApiCtx<'_>) -> api::Expression {
panic!("Cannot serialize expr!")
}
let f = mtreev_to_expr(f, stack.push(SrcToExprStep::Left), ctx).boxed_local().await;
ExprKind::Call(f.at(Pos::None), x_kind.at(x.pos.clone()))
}

22
orchid-host/src/expr_store.rs
View File

@@ -1,5 +1,6 @@
use std::cell::RefCell;
use std::fmt;
use std::rc::Rc;
use hashbrown::HashMap;
use hashbrown::hash_map::Entry;
@@ -8,10 +9,19 @@ use crate::api;
use crate::expr::Expr;
#[derive(Default)]
pub struct ExprStore(RefCell<HashMap<api::ExprTicket, (u32, Expr)>>);
pub struct ExprStoreData {
exprs: RefCell<HashMap<api::ExprTicket, (u32, Expr)>>,
parent: Option<ExprStore>,
}
#[derive(Clone, Default)]
pub struct ExprStore(Rc<ExprStoreData>);
impl ExprStore {
#[must_use]
pub fn derive(&self) -> Self {
Self(Rc::new(ExprStoreData { exprs: RefCell::default(), parent: Some(self.clone()) }))
}
pub fn give_expr(&self, expr: Expr) {
match self.0.borrow_mut().entry(expr.id()) {
match self.0.exprs.borrow_mut().entry(expr.id()) {
Entry::Occupied(mut oe) => oe.get_mut().0 += 1,
Entry::Vacant(v) => {
v.insert((1, expr));
@@ -19,16 +29,18 @@ impl ExprStore {
}
}
pub fn take_expr(&self, ticket: api::ExprTicket) {
(self.0.borrow_mut().entry(ticket))
(self.0.exprs.borrow_mut().entry(ticket))
.and_replace_entry_with(|_, (rc, rt)| (1 < rc).then_some((rc - 1, rt)));
}
#[must_use]
pub fn get_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
self.0.borrow().get(&ticket).map(|(_, expr)| expr.clone())
(self.0.exprs.borrow().get(&ticket).map(|(_, expr)| expr.clone()))
.or_else(|| self.0.parent.as_ref()?.get_expr(ticket))
}
}
impl fmt::Display for ExprStore {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let r = self.0.borrow();
let r = self.0.exprs.borrow();
let rc: u32 = r.values().map(|v| v.0).sum();
write!(f, "Store holding {rc} refs to {} exprs", r.len())
}

314
orchid-host/src/extension.rs
View File

@@ -2,29 +2,33 @@ use std::cell::RefCell;
use std::future::Future;
use std::io;
use std::num::NonZeroU64;
use std::pin::pin;
use std::rc::{Rc, Weak};
use async_std::channel::{self, Sender};
use async_std::sync::Mutex;
use async_stream::stream;
use derive_destructure::destructure;
use futures::FutureExt;
use futures::future::{join, join_all};
use futures::{StreamExt, stream};
use hashbrown::HashMap;
use itertools::Itertools;
use orchid_api::HostMsgSet;
use orchid_api_traits::Request;
use orchid_base::builtin::ExtInit;
use orchid_base::clone;
use orchid_base::format::{FmtCtxImpl, Format};
use orchid_base::interner::Tok;
use orchid_base::logging::Logger;
use orchid_base::name::Sym;
use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
use crate::api;
use crate::atom::AtomHand;
use crate::ctx::Ctx;
use crate::dealias::{ChildError, ChildErrorKind, walk};
use crate::expr_store::ExprStore;
use crate::system::SystemCtor;
use crate::tree::MemberKind;
pub struct ReqPair<R: Request>(R, Sender<R::Response>);
@@ -35,19 +39,22 @@ pub struct ReqPair<R: Request>(R, Sender<R::Response>);
#[derive(destructure)]
pub struct ExtensionData {
ctx: Ctx,
init: ExtInit,
reqnot: ReqNot<api::HostMsgSet>,
systems: Vec<SystemCtor>,
logger: Logger,
next_pars: RefCell<NonZeroU64>,
exprs: ExprStore,
exiting_snd: Sender<()>,
lex_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::SubLex>>>>,
mac_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::RunMacros>>>>,
}
impl Drop for ExtensionData {
fn drop(&mut self) {
let reqnot = self.reqnot.clone();
(self.ctx.spawn)(Box::pin(async move { reqnot.notify(api::HostExtNotif::Exit).await }))
let exiting_snd = self.exiting_snd.clone();
(self.ctx.spawn)(Box::pin(async move {
reqnot.notify(api::HostExtNotif::Exit).await;
exiting_snd.send(()).await.unwrap()
}))
}
}
@@ -55,121 +62,181 @@ impl Drop for ExtensionData {
pub struct Extension(Rc<ExtensionData>);
impl Extension {
pub fn new(init: ExtInit, logger: Logger, msg_logger: Logger, ctx: Ctx) -> io::Result<Self> {
Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
exprs: ExprStore::default(),
ctx: ctx.clone(),
systems: (init.systems.iter().cloned())
.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
.collect(),
logger: logger.clone(),
init,
next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
lex_recur: Mutex::default(),
mac_recur: Mutex::default(),
reqnot: ReqNot::new(
msg_logger,
clone!(weak; move |sfn, _| clone!(weak; async move {
let data = weak.upgrade().unwrap();
data.init.send(sfn).await
}.boxed_local())),
clone!(weak; move |notif, _| {
clone!(weak; Box::pin(async move {
let this = Extension(weak.upgrade().unwrap());
match notif {
api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket");
this.0.exprs.give_expr(target)
}
api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
this.assert_own_sys(rel.0).await;
this.0.exprs.take_expr(rel.1)
}
api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
this.assert_own_sys(mov.dec).await;
let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id");
let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket");
recp.ext().0.exprs.give_expr(expr);
this.0.exprs.take_expr(mov.expr);
},
api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str),
Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| {
let init = Rc::new(init);
let (exiting_snd, exiting_rcv) = channel::bounded::<()>(1);
(ctx.spawn)(clone!(init, weak, ctx; Box::pin(async move {
let rcv_stream = stream! { loop { yield init.recv().await } };
let mut event_stream = pin!(stream::select(exiting_rcv.map(|()| None), rcv_stream));
while let Some(Some(msg)) = event_stream.next().await {
if let Some(reqnot) = weak.upgrade().map(|rc| rc.reqnot.clone()) {
let reqnot = reqnot.clone();
(ctx.spawn)(Box::pin(async move {
reqnot.receive(&msg).await;
}))
}
}))}),
{
clone!(weak, ctx);
move |hand, req| {
clone!(weak, ctx);
Box::pin(async move {
let this = Self(weak.upgrade().unwrap());
writeln!(this.reqnot().logger(), "Host received request {req:?}");
let i = this.ctx().i.clone();
match req {
api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
api::ExtHostReq::IntReq(intreq) => match intreq {
api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await,
api::IntReq::InternStrv(v) => {
let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await;
hand.handle(&v, &i.i(&tokens).await.to_api()).await
},
api::IntReq::ExternStr(si) =>
hand.handle(&si, &Tok::<String>::from_api(si.0, &i).await.rc()).await,
api::IntReq::ExternStrv(vi) => {
let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec();
hand.handle(&vi, &markerv).await
},
},
api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
let sys = ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
let reply =
sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
hand.handle(fw, &reply).await
},
api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
let sys = ctx.system_inst(id).await.unwrap();
hand.handle(fw, &sys.request(body.clone()).await).await
},
api::ExtHostReq::SubLex(sl) => {
let (rep_in, rep_out) = channel::bounded(1);
{
let lex_g = this.0.lex_recur.lock().await;
let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
}
hand.handle(&sl, &rep_out.recv().await.unwrap()).await
},
api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => {
let expr = this.exprs().get_expr(target).expect("Invalid ticket");
hand
.handle(&ins, &api::Inspected {
refcount: expr.strong_count() as u32,
location: expr.pos().to_api(),
kind: expr.to_api().await,
})
.await
},
api::ExtHostReq::RunMacros(rm) => {
let (rep_in, rep_out) = channel::bounded(1);
let lex_g = this.0.mac_recur.lock().await;
let req_in = lex_g.get(&rm.run_id).expect("Sublex for nonexistent lexid");
req_in.send(ReqPair(rm.clone(), rep_in)).await.unwrap();
hand.handle(&rm, &rep_out.recv().await.unwrap()).await
},
api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
let atom = AtomHand::new(atom.clone(), &ctx).await;
let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
hand.handle(eap, &unit.to_api()).await
},
}
})));
ExtensionData {
exiting_snd,
exprs: ctx.common_exprs.derive(),
ctx: ctx.clone(),
systems: (init.systems.iter().cloned())
.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
.collect(),
logger: logger.clone(),
next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
lex_recur: Mutex::default(),
reqnot: ReqNot::new(
msg_logger,
move |sfn, _| clone!(init; Box::pin(async move { init.send(sfn).await })),
clone!(weak; move |notif, _| {
clone!(weak; Box::pin(async move {
let this = Extension(weak.upgrade().unwrap());
match notif {
api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket");
this.0.exprs.give_expr(target)
}
})
}
},
),
api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
this.assert_own_sys(rel.0).await;
this.0.exprs.take_expr(rel.1)
}
api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
this.assert_own_sys(mov.dec).await;
let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id");
let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket");
recp.ext().0.exprs.give_expr(expr);
this.0.exprs.take_expr(mov.expr);
},
api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str),
}
}))}),
{
clone!(weak, ctx);
move |hand, req| {
clone!(weak, ctx);
Box::pin(async move {
let this = Self(weak.upgrade().unwrap());
writeln!(this.reqnot().logger(), "Host received request {req:?}");
let i = this.ctx().i.clone();
match req {
api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
api::ExtHostReq::IntReq(intreq) => match intreq {
api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await,
api::IntReq::InternStrv(v) => {
let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await;
hand.handle(&v, &i.i(&tokens).await.to_api()).await
},
api::IntReq::ExternStr(si) =>
hand.handle(&si, &Tok::<String>::from_api(si.0, &i).await.rc()).await,
api::IntReq::ExternStrv(vi) => {
let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec();
hand.handle(&vi, &markerv).await
},
},
api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
let sys =
ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
let reply =
sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
hand.handle(fw, &reply).await
},
api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
let sys = ctx.system_inst(id).await.unwrap();
hand.handle(fw, &sys.request(body.clone()).await).await
},
api::ExtHostReq::SubLex(sl) => {
let (rep_in, rep_out) = channel::bounded(1);
{
let lex_g = this.0.lex_recur.lock().await;
let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
}
hand.handle(&sl, &rep_out.recv().await.unwrap()).await
},
api::ExtHostReq::ExprReq(api::ExprReq::Inspect(
ins @ api::Inspect { target },
)) => {
let expr = this.exprs().get_expr(target).expect("Invalid ticket");
hand
.handle(&ins, &api::Inspected {
refcount: expr.strong_count() as u32,
location: expr.pos().to_api(),
kind: expr.to_api().await,
})
.await
},
api::ExtHostReq::LsModule(ref ls @ api::LsModule(_sys, path)) => {
let reply: <api::LsModule as Request>::Response = 'reply: {
let path = i.ex(path).await;
let root = (ctx.root.read().await.upgrade())
.expect("LSModule called when root isn't in context");
let root_data = &mut *root.0.write().await;
let mut walk_ctx = (ctx.clone(), &mut root_data.consts);
let module =
match walk(&root_data.root, false, path.iter().cloned(), &mut walk_ctx)
.await
{
Ok(module) => module,
Err(ChildError { kind, .. }) =>
break 'reply Err(match kind {
ChildErrorKind::Private => panic!("Access checking was disabled"),
ChildErrorKind::Constant => api::LsModuleError::IsConstant,
ChildErrorKind::Missing => api::LsModuleError::InvalidPath,
}),
};
let mut members = std::collections::HashMap::new();
for (k, v) in &module.members {
let kind = match v.kind(ctx.clone(), &mut root_data.consts).await {
MemberKind::Const => api::MemberInfoKind::Constant,
MemberKind::Module(_) => api::MemberInfoKind::Module,
};
members.insert(k.to_api(), api::MemberInfo { public: v.public, kind });
}
Ok(api::ModuleInfo { members })
};
hand.handle(ls, &reply).await
},
api::ExtHostReq::ResolveNames(ref rn) => {
let api::ResolveNames { constid, names, sys } = rn;
let mut resolver = {
let systems = ctx.systems.read().await;
let weak_sys = systems.get(sys).expect("ResolveNames for invalid sys");
let sys = weak_sys.upgrade().expect("ResolveNames after sys drop");
sys.name_resolver(*constid).await
};
let mut responses = vec![const { None }; names.len()];
for (i, name) in names.iter().enumerate() {
if let Some(abs) = resolver(&ctx.i.ex(*name).await[..]).await {
responses[i] = Some(abs.to_sym(&ctx.i).await.to_api())
}
}
hand.handle(rn, &responses).await
},
api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
let atom = AtomHand::new(atom.clone(), &ctx).await;
let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
hand.handle(eap, &unit.to_api()).await
},
}
})
}
},
),
}
})))
}
pub(crate) fn reqnot(&self) -> &ReqNot<HostMsgSet> { &self.0.reqnot }
#[must_use]
pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.reqnot }
#[must_use]
pub fn ctx(&self) -> &Ctx { &self.0.ctx }
#[must_use]
pub fn logger(&self) -> &Logger { &self.0.logger }
pub fn system_ctors(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
#[must_use]
pub fn exprs(&self) -> &ExprStore { &self.0.exprs }
#[must_use]
pub async fn is_own_sys(&self, id: api::SysId) -> bool {
let sys = self.ctx().system_inst(id).await.expect("invalid sender sys id");
Rc::ptr_eq(&self.0, &sys.ext().0)
@@ -177,6 +244,7 @@ impl Extension {
pub async fn assert_own_sys(&self, id: api::SysId) {
assert!(self.is_own_sys(id).await, "Incoming message impersonates separate system");
}
#[must_use]
pub fn next_pars(&self) -> NonZeroU64 {
let mut next_pars = self.0.next_pars.borrow_mut();
*next_pars = next_pars.checked_add(1).unwrap_or(NonZeroU64::new(1).unwrap());
@@ -185,6 +253,7 @@ impl Extension {
pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
&self,
source: Tok<String>,
src: Sym,
pos: u32,
sys: api::SysId,
mut r: impl FnMut(u32) -> F,
@@ -196,8 +265,9 @@ impl Extension {
self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
let (ret, ()) = join(
async {
let res =
(self.reqnot()).request(api::LexExpr { id, pos, sys, text: source.to_api() }).await;
let res = (self.reqnot())
.request(api::LexExpr { id, pos, sys, src: src.to_api(), text: source.to_api() })
.await;
// collect sender to unblock recursion handler branch before returning
self.0.lex_recur.lock().await.remove(&id);
res
@@ -212,20 +282,6 @@ impl Extension {
.await;
ret.transpose()
}
pub async fn recv_one(&self) {
let reqnot = self.0.reqnot.clone();
let ctx = self.ctx().clone();
(self.0.init.recv(Box::new(move |msg| {
Box::pin(async move {
let msg = msg.to_vec();
let reqnot = reqnot.clone();
(ctx.spawn)(Box::pin(async move {
reqnot.receive(&msg).await;
}))
})
})))
.await;
}
pub fn system_drop(&self, id: api::SysId) {
let rc = self.clone();
(self.ctx().spawn)(Box::pin(async move {
@@ -233,10 +289,12 @@ impl Extension {
rc.ctx().systems.write().await.remove(&id);
}))
}
#[must_use]
pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }
}
pub struct WeakExtension(Weak<ExtensionData>);
impl WeakExtension {
#[must_use]
pub fn upgrade(&self) -> Option<Extension> { self.0.upgrade().map(Extension) }
}

138
orchid-host/src/lex.rs
View File

@@ -1,48 +1,50 @@
use std::num::NonZeroU64;
use std::sync::Arc;
use std::rc::Rc;
use async_std::sync::Mutex;
use futures::FutureExt;
use hashbrown::HashMap;
use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
use orchid_base::interner::Tok;
use orchid_base::location::Pos;
use orchid_base::match_mapping;
use orchid_base::location::SrcRange;
use orchid_base::name::Sym;
use orchid_base::number::{num_to_err, parse_num};
use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
use orchid_base::tokens::PARENS;
use orchid_base::tree::{AtomRepr, Ph};
use orchid_base::tree::recur;
use crate::api;
use crate::atom::AtomHand;
use crate::ctx::Ctx;
use crate::expr::{Expr, ExprParseCtx, ExprWillPanic};
use crate::parsed::{ParsTok, ParsTokTree};
use crate::system::System;
use crate::tree::{ParsTok, ParsTokTree};
pub struct LexCtx<'a> {
pub systems: &'a [System],
pub source: &'a Tok<String>,
pub path: &'a Sym,
pub tail: &'a str,
pub sub_trees: &'a mut HashMap<api::TreeTicket, ParsTokTree>,
pub sub_trees: &'a mut Vec<Expr>,
pub ctx: &'a Ctx,
}
impl<'a> LexCtx<'a> {
#[must_use]
pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
where 'a: 'b {
LexCtx {
source: self.source,
path: self.path,
tail: &self.source[pos as usize..],
systems: self.systems,
sub_trees: &mut *self.sub_trees,
ctx: self.ctx,
}
}
#[must_use]
pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 }
#[must_use]
pub fn end_pos(&self) -> u32 { self.source.len() as u32 }
pub fn set_pos(&mut self, pos: u32) { self.tail = &self.source[pos as usize..] }
pub fn push_pos(&mut self, delta: u32) { self.set_pos(self.get_pos() + delta) }
pub fn set_tail(&mut self, tail: &'a str) { self.tail = tail }
#[must_use]
pub fn strip_prefix(&mut self, tgt: &str) -> bool {
if let Some(src) = self.tail.strip_prefix(tgt) {
self.tail = src;
@@ -50,14 +52,29 @@ impl<'a> LexCtx<'a> {
}
false
}
pub fn add_subtree(&mut self, subtree: ParsTokTree) -> api::TreeTicket {
let next_idx = api::TreeTicket(NonZeroU64::new(self.sub_trees.len() as u64 + 1).unwrap());
self.sub_trees.insert(next_idx, subtree);
next_idx
#[must_use]
pub async fn ser_subtree(&mut self, subtree: ParsTokTree) -> api::TokenTree {
let mut exprs = self.ctx.common_exprs.clone();
let without_new_expr = recur(subtree, &|tt, r| {
if let ParsTok::NewExpr(expr) = tt.tok {
return ParsTok::Handle(expr).at(tt.sr);
}
r(tt)
});
without_new_expr.into_api(&mut exprs, &mut ExprWillPanic).await
}
pub fn rm_subtree(&mut self, ticket: api::TreeTicket) -> ParsTokTree {
self.sub_trees.remove(&ticket).unwrap()
#[must_use]
pub async fn des_subtree(&mut self, tree: &api::TokenTree) -> ParsTokTree {
ParsTokTree::from_api(
tree,
&mut self.ctx.common_exprs.clone(),
&mut ExprParseCtx { ctx: self.ctx, exprs: &self.ctx.common_exprs },
self.path,
&self.ctx.i,
)
.await
}
#[must_use]
pub fn strip_char(&mut self, tgt: char) -> bool {
if let Some(src) = self.tail.strip_prefix(tgt) {
self.tail = src;
@@ -69,6 +86,7 @@ impl<'a> LexCtx<'a> {
self.tail = self.tail.trim_start_matches(filter);
}
pub fn trim_ws(&mut self) { self.trim(|c| c.is_whitespace() && !"\r\n".contains(c)) }
#[must_use]
pub fn get_start_matches(&mut self, filter: impl Fn(char) -> bool) -> &'a str {
let rest = self.tail.trim_start_matches(filter);
let matches = &self.tail[..self.tail.len() - rest.len()];
@@ -86,22 +104,27 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
);
let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
ParsTok::BR
} else if ctx.strip_prefix("::") {
ParsTok::NS
} else if let Some(tail) = (ctx.tail.starts_with(name_start).then_some(ctx.tail))
.and_then(|t| t.trim_start_matches(name_char).strip_prefix("::"))
{
let name = &ctx.tail[..ctx.tail.len() - tail.len() - "::".len()];
ctx.set_tail(tail);
let body = lex_once(ctx).boxed_local().await?;
ParsTok::NS(ctx.ctx.i.i(name).await, Box::new(body))
} else if ctx.strip_prefix("--[") {
let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
return Err(mk_errv(
ctx.ctx.i.i("Unterminated block comment").await,
"This block comment has no ending ]--",
[Pos::Range(start..start + 3).into()],
[SrcRange::new(start..start + 3, ctx.path)],
));
};
ctx.set_tail(tail);
ParsTok::Comment(Arc::new(cmt.to_string()))
ParsTok::Comment(Rc::new(cmt.to_string()))
} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
ctx.push_pos(end as u32);
ParsTok::Comment(Arc::new(tail[2..end].to_string()))
ParsTok::Comment(Rc::new(tail[2..end].to_string()))
} else if ctx.strip_char('\\') {
let mut arg = Vec::new();
ctx.trim_ws();
@@ -110,7 +133,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
return Err(mk_errv(
ctx.ctx.i.i("Unclosed lambda").await,
"Lambdae started with \\ should separate arguments from body with .",
[Pos::Range(start..start + 1).into()],
[SrcRange::new(start..start + 1, ctx.path)],
));
}
arg.push(lex_once(ctx).boxed_local().await?);
@@ -125,39 +148,25 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
return Err(mk_errv(
ctx.ctx.i.i("unclosed paren").await,
format!("this {lp} has no matching {rp}"),
[Pos::Range(start..start + 1).into()],
[SrcRange::new(start..start + 1, ctx.path)],
));
}
body.push(lex_once(ctx).boxed_local().await?);
ctx.trim_ws();
}
ParsTok::S(*paren, body)
} else if ctx.strip_prefix("macro")
&& !ctx.tail.chars().next().is_some_and(|x| x.is_ascii_alphabetic())
{
ctx.strip_prefix("macro");
if ctx.strip_char('(') {
let pos = ctx.get_pos();
let numstr = ctx.get_start_matches(|x| x != ')').trim();
match parse_num(numstr) {
Ok(num) => ParsTok::Macro(Some(num.to_f64())),
Err(e) => return Err(num_to_err(e, pos, &ctx.ctx.i).await.into()),
}
} else {
ParsTok::Macro(None)
}
} else {
for sys in ctx.systems {
let mut errors = Vec::new();
if ctx.tail.starts_with(|c| sys.can_lex(c)) {
let (source, pos) = (ctx.source.clone(), ctx.get_pos());
let (source, pos, path) = (ctx.source.clone(), ctx.get_pos(), ctx.path.clone());
let ctx_lck = &Mutex::new(&mut *ctx);
let errors_lck = &Mutex::new(&mut errors);
let lx = sys
.lex(source, pos, |pos| async move {
.lex(source, path, pos, |pos| async move {
let mut ctx_g = ctx_lck.lock().await;
match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
Ok(t) => Some(api::SubLexed { pos: t.range.end, ticket: ctx_g.add_subtree(t) }),
Ok(t) => Some(api::SubLexed { pos: t.sr.end(), tree: ctx_g.ser_subtree(t).await }),
Err(e) => {
errors_lck.lock().await.push(e);
None
@@ -172,7 +181,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
),
Ok(Some(lexed)) => {
ctx.set_pos(lexed.pos);
return Ok(tt_to_owned(&lexed.expr, ctx).await);
return Ok(ctx.des_subtree(&lexed.expr).await);
},
Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
Some(errors) => return Err(errors),
@@ -189,47 +198,22 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
return Err(mk_errv(
ctx.ctx.i.i("Unrecognized character").await,
"The following syntax is meaningless.",
[Pos::Range(start..start + 1).into()],
[SrcRange::new(start..start + 1, ctx.path)],
));
}
};
Ok(ParsTokTree { tok, range: start..ctx.get_pos() })
Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
}
async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
let tok = match_mapping!(&api.token, api::Token => ParsTok {
Atom(atom =>
AtomHand::from_api(atom, Pos::Range(api.range.clone()), &mut ctx.ctx.clone()).await
),
Bottom(err => OrcErrv::from_api(err, &ctx.ctx.i).await),
LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await),
Name(name => Tok::from_api(*name, &ctx.ctx.i).await),
Reference(tstrv => Sym::from_api(*tstrv, &ctx.ctx.i).await),
S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await),
BR, NS,
Comment(c.clone()),
Ph(ph => Ph::from_api(ph, &ctx.ctx.i).await),
Macro(*prio),
} {
api::Token::Slot(id) => return ctx.rm_subtree(*id),
});
ParsTokTree { range: api.range.clone(), tok }
}
async fn ttv_to_owned<'a>(
api: impl IntoIterator<Item = &'a api::TokenTree>,
ctx: &mut LexCtx<'_>,
) -> Vec<ParsTokTree> {
let mut out = Vec::new();
for tt in api {
out.push(tt_to_owned(tt, ctx).await)
}
out
}
pub async fn lex(text: Tok<String>, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
let mut sub_trees = HashMap::new();
let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, ctx };
pub async fn lex(
text: Tok<String>,
path: Sym,
systems: &[System],
ctx: &Ctx,
) -> OrcRes<Vec<ParsTokTree>> {
let mut sub_trees = Vec::new();
let mut ctx =
LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };
let mut tokv = Vec::new();
ctx.trim(unrep_space);
while !ctx.tail.is_empty() {

3
orchid-host/src/lib.rs
View File

@@ -8,9 +8,8 @@ pub mod expr;
pub mod expr_store;
pub mod extension;
pub mod lex;
pub mod macros;
pub mod parse;
pub mod rule;
pub mod parsed;
pub mod subprocess;
pub mod system;
pub mod tree;

103
orchid-host/src/macros.rs
View File

@@ -1,103 +0,0 @@
use std::rc::Rc;
use futures::FutureExt;
use hashbrown::{HashMap, HashSet};
use itertools::Itertools;
use orchid_base::clone;
use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api};
use orchid_base::name::Sym;
use trait_set::trait_set;
use crate::api;
use crate::atom::AtomHand;
use crate::ctx::Ctx;
use crate::rule::state::MatchState;
use crate::tree::Code;
pub type MacTok = MTok<'static, AtomHand>;
pub type MacTree = MTree<'static, AtomHand>;
trait_set! {
trait MacroCB = Fn(Vec<MacTree>) -> Option<Vec<MacTree>>;
}
type Slots = HashMap<api::MacroTreeId, Rc<MacTok>>;
pub async fn macro_treev_to_api(mtree: Vec<MacTree>, slots: &mut Slots) -> Vec<api::MacroTree> {
mtreev_to_api(&mtree, &mut |a: &AtomHand| {
let id = api::MacroTreeId((slots.len() as u64 + 1).try_into().unwrap());
slots.insert(id, Rc::new(MacTok::Atom(a.clone())));
async move { api::MacroToken::Slot(id) }.boxed_local()
})
.await
}
pub async fn macro_treev_from_api(api: Vec<api::MacroTree>, ctx: Ctx) -> Vec<MacTree> {
mtreev_from_api(&api, &ctx.clone().i, &mut move |atom| {
clone!(ctx);
Box::pin(async move { MacTok::Atom(AtomHand::new(atom.clone(), &ctx).await) })
})
.await
}
pub fn deslot_macro(tree: &[MacTree], slots: &mut Slots) -> Option<Vec<MacTree>> {
return work(slots, tree);
fn work(
slots: &mut HashMap<api::MacroTreeId, Rc<MacTok>>,
tree: &[MacTree],
) -> Option<Vec<MacTree>> {
let items = (tree.iter())
.map(|t| {
Some(MacTree {
tok: match &*t.tok {
MacTok::Atom(_) | MacTok::Name(_) | MacTok::Ph(_) => return None,
MacTok::Ref(_) => panic!("Ref is an extension-local optimization"),
MacTok::Done(_) => panic!("Created and removed by matcher"),
MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(),
MacTok::S(paren, b) => Rc::new(MacTok::S(*paren, work(slots, b)?)),
MacTok::Lambda(a, b) => Rc::new(match (work(slots, a), work(slots, b)) {
(None, None) => return None,
(Some(a), None) => MacTok::Lambda(a, b.clone()),
(None, Some(b)) => MacTok::Lambda(a.clone(), b),
(Some(a), Some(b)) => MacTok::Lambda(a, b),
}),
},
pos: t.pos.clone(),
})
})
.collect_vec();
let any_changed = items.iter().any(Option::is_some);
any_changed.then(|| {
(items.into_iter().enumerate())
.map(|(i, opt)| opt.unwrap_or_else(|| tree[i].clone()))
.collect_vec()
})
}
}
pub struct Macro<Matcher> {
deps: HashSet<Sym>,
cases: Vec<(Matcher, Code)>,
}
fn fill_lexicon(tgt: &MacTree, lexicon: &mut HashSet<Sym>) {
match &*tgt.tok {
MTok::Name(n) => {
lexicon.insert(n.clone());
},
MTok::Lambda(arg, body) => {
arg.iter().for_each(|t| fill_lexicon(t, lexicon));
body.iter().for_each(|t| fill_lexicon(t, lexicon))
},
MTok::S(_, body) => body.iter().for_each(|t| fill_lexicon(t, lexicon)),
_ => (),
}
}
fn run_body(body: &Code, mut state: MatchState<'_>) -> Vec<MacTree> {
let inject: Vec<MacTree> = todo!("Call the interpreter with bindings");
inject
.into_iter()
.map(|MTree { pos, tok }| MTree { pos, tok: Rc::new(MTok::Done(tok)) })
.collect_vec()
}

387
orchid-host/src/parse.rs
View File

@@ -1,317 +1,262 @@
use std::rc::Rc;
use futures::FutureExt;
use futures::future::join_all;
use itertools::Itertools;
use never::Never;
use orchid_base::error::{OrcErrv, OrcRes, Reporter, ReporterImpl, mk_err, mk_errv};
use orchid_base::interner::Tok;
use orchid_base::location::Pos;
use orchid_base::macros::{MTok, MTree};
use orchid_base::name::Sym;
use orchid_base::error::{OrcRes, Reporter, mk_errv};
use orchid_base::format::fmt;
use orchid_base::interner::{Interner, Tok};
use orchid_base::name::{Sym, VPath};
use orchid_base::parse::{
Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, try_pop_no_fluff,
Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
try_pop_no_fluff,
};
use orchid_base::tree::{Paren, TokTree, Token};
use substack::Substack;
use crate::atom::AtomHand;
use crate::macros::MacTree;
use crate::ctx::Ctx;
use crate::expr::{Expr, ExprKind, PathSetBuilder};
use crate::parsed::{Item, ItemKind, ParsedMember, ParsedMemberKind, ParsedModule};
use crate::system::System;
use crate::tree::{Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, Rule, RuleKind};
type ParsSnippet<'a> = Snippet<'a, 'static, AtomHand, Never>;
type ParsSnippet<'a> = Snippet<'a, Expr, Expr>;
pub struct ParseCtxImpl<'a> {
pub struct HostParseCtxImpl<'a> {
pub ctx: Ctx,
pub src: Sym,
pub systems: &'a [System],
pub reporter: &'a ReporterImpl,
pub rep: &'a Reporter,
}
impl ParseCtx for ParseCtxImpl<'_> {
fn reporter(&self) -> &(impl Reporter + ?Sized) { self.reporter }
impl ParseCtx for HostParseCtxImpl<'_> {
fn reporter(&self) -> &Reporter { self.rep }
fn i(&self) -> &Interner { &self.ctx.i }
}
impl HostParseCtx for HostParseCtxImpl<'_> {
fn ctx(&self) -> &Ctx { &self.ctx }
fn systems(&self) -> impl Iterator<Item = &System> { self.systems.iter() }
}
pub trait ParseCtx {
pub trait HostParseCtx: ParseCtx {
#[must_use]
fn ctx(&self) -> &Ctx;
#[must_use]
fn systems(&self) -> impl Iterator<Item = &System>;
fn reporter(&self) -> &(impl Reporter + ?Sized);
}
pub async fn parse_items(
ctx: &impl ParseCtx,
ctx: &impl HostParseCtx,
path: Substack<'_, Tok<String>>,
items: ParsSnippet<'_>,
) -> OrcRes<Vec<Item>> {
let lines = line_items(items).await;
let lines = line_items(ctx, items).await;
let line_res =
join_all(lines.into_iter().map(|p| parse_item(ctx, path.clone(), p.output, p.tail))).await;
Ok(line_res.into_iter().flat_map(|l| l.ok().into_iter().flatten()).collect())
}
pub async fn parse_item(
ctx: &impl ParseCtx,
ctx: &impl HostParseCtx,
path: Substack<'_, Tok<String>>,
comments: Vec<Comment>,
item: ParsSnippet<'_>,
) -> OrcRes<Vec<Item>> {
match item.pop_front() {
Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
n if *n == item.i().i("export").await => match try_pop_no_fluff(postdisc).await? {
n if *n == ctx.i().i("export").await => match try_pop_no_fluff(ctx, postdisc).await? {
Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
Parsed { output: TokTree { tok: Token::NS, .. }, tail } => {
let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail).await?;
let mut ok = Vec::new();
for (e, pos) in exports {
match (&e.path[..], e.name) {
([], Some(n)) =>
ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
(_, Some(_)) => ctx.reporter().report(mk_err(
tail.i().i("Compound export").await,
"Cannot export compound names (names containing the :: separator)",
[pos.into()],
)),
(_, None) => ctx.reporter().report(mk_err(
tail.i().i("Wildcard export").await,
"Exports cannot contain the globstar *",
[pos.into()],
)),
}
}
expect_end(tail).await?;
Ok(ok)
},
Parsed { output, tail } => Err(mk_errv(
tail.i().i("Malformed export").await,
"`export` can either prefix other lines or list names inside ::( ) or ::[ ]",
[Pos::Range(output.range.clone()).into()],
Parsed { output, tail: _ } => Err(mk_errv(
ctx.i().i("Malformed export").await,
"`export` can either prefix other lines or list names inside ( )",
[output.sr()],
)),
},
n if *n == item.i().i("import").await => parse_import(ctx, postdisc).await.map(|v| {
Vec::from_iter(v.into_iter().map(|(t, pos)| Item {
n if *n == ctx.i().i("import").await => {
let imports = parse_import(ctx, postdisc).await?;
Ok(Vec::from_iter(imports.into_iter().map(|t| Item {
comments: comments.clone(),
pos,
sr: t.sr.clone(),
kind: ItemKind::Import(t),
}))
}),
})))
},
n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
},
Some(_) => Err(mk_errv(
item.i().i("Expected a line type").await,
ctx.i().i("Expected a line type").await,
"All lines must begin with a keyword",
[Pos::Range(item.pos()).into()],
[item.sr()],
)),
None => unreachable!("These lines are filtered and aggregated in earlier stages"),
}
}
pub async fn parse_import(
ctx: &impl ParseCtx,
tail: ParsSnippet<'_>,
) -> OrcRes<Vec<(Import, Pos)>> {
let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail).await?;
expect_end(tail).await?;
pub async fn parse_import<'a>(
ctx: &impl HostParseCtx,
tail: ParsSnippet<'a>,
) -> OrcRes<Vec<Import>> {
let Parsed { output: imports, tail } = parse_multiname(ctx, tail).await?;
expect_end(ctx, tail).await?;
Ok(imports)
}
pub async fn parse_exportable_item(
ctx: &impl ParseCtx,
pub async fn parse_exportable_item<'a>(
ctx: &impl HostParseCtx,
path: Substack<'_, Tok<String>>,
comments: Vec<Comment>,
exported: bool,
discr: Tok<String>,
tail: ParsSnippet<'_>,
tail: ParsSnippet<'a>,
) -> OrcRes<Vec<Item>> {
let kind = if discr == tail.i().i("mod").await {
let kind = if discr == ctx.i().i("mod").await {
let (name, body) = parse_module(ctx, path, tail).await?;
ItemKind::Member(Member::new(name, MemberKind::Mod(body)))
} else if discr == tail.i().i("const").await {
let (name, val) = parse_const(tail, path.clone()).await?;
let locator = CodeLocator::to_const(tail.i().i(&path.push(name.clone()).unreverse()).await);
ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val))))
ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::Mod(body) })
} else if discr == ctx.i().i("const").await {
let (name, expr) = parse_const(ctx, tail, path.clone()).await?;
ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::ParsedConst(expr) })
} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
let line = sys.parse(tail.to_vec(), exported, comments).await?;
return parse_items(ctx, path, Snippet::new(tail.prev(), &line, tail.i())).await;
return sys
.parse(path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
let source = Snippet::new(lines.first().unwrap(), &lines);
parse_items(ctx, stack, source).await
})
.await;
} else {
let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
return Err(mk_errv(
tail.i().i("Unrecognized line type").await,
ctx.i().i("Unrecognized line type").await,
format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
[Pos::Range(tail.prev().range.clone()).into()],
[tail.prev().sr()],
));
};
Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }])
Ok(vec![Item { comments, sr: tail.sr(), kind }])
}
pub async fn parse_module(
ctx: &impl ParseCtx,
pub async fn parse_module<'a>(
ctx: &impl HostParseCtx,
path: Substack<'_, Tok<String>>,
tail: ParsSnippet<'_>,
) -> OrcRes<(Tok<String>, Module)> {
let (name, tail) = match try_pop_no_fluff(tail).await? {
tail: ParsSnippet<'a>,
) -> OrcRes<(Tok<String>, ParsedModule)> {
let (name, tail) = match try_pop_no_fluff(ctx, tail).await? {
Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
Parsed { output, .. } => {
return Err(mk_errv(
tail.i().i("Missing module name").await,
format!("A name was expected, {} was found", tail.fmt(output).await),
[Pos::Range(output.range.clone()).into()],
ctx.i().i("Missing module name").await,
format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
[output.sr()],
));
},
};
let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?;
expect_end(surplus).await?;
let Some(body) = output.as_s(Paren::Round, tail.i()) else {
let Parsed { output, tail: surplus } = try_pop_no_fluff(ctx, tail).await?;
expect_end(ctx, surplus).await?;
let Some(body) = output.as_s(Paren::Round) else {
return Err(mk_errv(
tail.i().i("Expected module body").await,
format!("A ( block ) was expected, {} was found", tail.fmt(output).await),
[Pos::Range(output.range.clone()).into()],
ctx.i().i("Expected module body").await,
format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await),
[output.sr()],
));
};
let path = path.push(name.clone());
Ok((name, Module::new(parse_items(ctx, path, body).await?)))
Ok((name, ParsedModule::new(parse_items(ctx, path, body).await?)))
}
pub async fn parse_const(
tail: ParsSnippet<'_>,
pub async fn parse_const<'a>(
ctx: &impl HostParseCtx,
tail: ParsSnippet<'a>,
path: Substack<'_, Tok<String>>,
) -> OrcRes<(Tok<String>, Vec<MacTree>)> {
let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
) -> OrcRes<(Tok<String>, Expr)> {
let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?;
let Some(name) = output.as_name() else {
return Err(mk_errv(
tail.i().i("Missing module name").await,
format!("A name was expected, {} was found", tail.fmt(output).await),
[Pos::Range(output.range.clone()).into()],
ctx.i().i("Missing module name").await,
format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
[output.sr()],
));
};
let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
if !output.is_kw(tail.i().i("=").await) {
let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?;
if !output.is_kw(ctx.i().i("=").await) {
return Err(mk_errv(
tail.i().i("Missing = separator").await,
format!("Expected = , found {}", tail.fmt(output).await),
[Pos::Range(output.range.clone()).into()],
ctx.i().i("Missing = separator").await,
format!("Expected = , found {}", fmt(output, ctx.i()).await),
[output.sr()],
));
}
try_pop_no_fluff(tail).await?;
Ok((name, parse_mtree(tail, path).await?))
try_pop_no_fluff(ctx, tail).await?;
// ctx.save_const(path, tail[..].to_vec()).await;
let final_path =
VPath::new(path.unreverse()).name_with_suffix(name.clone()).to_sym(ctx.i()).await;
let val = parse_expr(ctx, final_path, PathSetBuilder::new(), tail).await?;
Ok((name, val))
}
pub async fn parse_mtree(
mut snip: ParsSnippet<'_>,
path: Substack<'_, Tok<String>>,
) -> OrcRes<Vec<MacTree>> {
let mut mtreev = Vec::new();
while let Some((ttree, tail)) = snip.pop_front() {
snip = tail;
let (range, tok, tail) = match &ttree.tok {
Token::S(p, b) => {
let b = parse_mtree(Snippet::new(ttree, b, snip.i()), path.clone()).boxed_local().await?;
(ttree.range.clone(), MTok::S(*p, b), tail)
},
Token::Reference(name) => (ttree.range.clone(), MTok::Name(name.clone()), tail),
Token::Name(tok) => {
let mut segments = path.unreverse();
segments.push(tok.clone());
let mut end = ttree.range.end;
while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() {
let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
let Some(seg) = output.as_name() else {
return Err(mk_errv(
tail.i().i("Namespaced name interrupted").await,
"In expression context, :: must always be followed by a name.\n\
::() is permitted only in import and export items",
[Pos::Range(output.range.clone()).into()],
));
};
segments.push(seg);
snip = tail;
end = output.range.end;
}
(ttree.range.start..end, MTok::Name(Sym::new(segments, snip.i()).await.unwrap()), snip)
},
Token::NS => {
return Err(mk_errv(
tail.i().i("Unexpected :: in expression").await,
":: can only follow a name",
[Pos::Range(ttree.range.clone()).into()],
));
},
Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail),
Token::Macro(_) => {
return Err(mk_errv(
tail.i().i("Invalid keyword in expression").await,
"Expressions cannot use `macro` as a name.",
[Pos::Range(ttree.range.clone()).into()],
));
},
Token::Atom(a) => (ttree.range.clone(), MTok::Atom(a.clone()), tail),
Token::BR | Token::Comment(_) => continue,
Token::Bottom(e) => return Err(e.clone()),
Token::LambdaHead(arg) => (
ttree.range.start..snip.pos().end,
MTok::Lambda(
parse_mtree(Snippet::new(ttree, arg, snip.i()), path.clone()).boxed_local().await?,
parse_mtree(tail, path.clone()).boxed_local().await?,
),
Snippet::new(ttree, &[], snip.i()),
),
Token::Slot(_) | Token::X(_) =>
panic!("Did not expect {} in parsed token tree", tail.fmt(ttree).await),
};
mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Rc::new(tok) });
snip = tail;
}
Ok(mtreev)
}
pub async fn parse_macro(
pub async fn parse_expr(
ctx: &impl HostParseCtx,
path: Sym,
psb: PathSetBuilder<'_, Tok<String>>,
tail: ParsSnippet<'_>,
macro_i: u16,
path: Substack<'_, Tok<String>>,
) -> OrcRes<Vec<Rule>> {
let (surplus, prev, block) = match try_pop_no_fluff(tail).await? {
Parsed { tail, output: o @ TokTree { tok: Token::S(Paren::Round, b), .. } } => (tail, o, b),
Parsed { output, .. } => {
return Err(mk_errv(
tail.i().i("m").await,
"Macro blocks must either start with a block or a ..$:number",
[Pos::Range(output.range.clone()).into()],
));
},
) -> OrcRes<Expr> {
let Some((last_idx, _)) = (tail.iter().enumerate().find(|(_, tt)| tt.as_lambda().is_some()))
.or_else(|| tail.iter().enumerate().rev().find(|(_, tt)| !tt.is_fluff()))
else {
return Err(mk_errv(ctx.i().i("Empty expression").await, "Expression ends abruptly here", [
tail.sr(),
]));
};
expect_end(surplus).await?;
let mut errors = Vec::new();
let mut rules = Vec::new();
for (i, item) in line_items(Snippet::new(prev, block, tail.i())).await.into_iter().enumerate() {
let Parsed { tail, output } = try_pop_no_fluff(item.tail).await?;
if !output.is_kw(tail.i().i("rule").await) {
errors.extend(mk_errv(
tail.i().i("non-rule in macro").await,
format!("Expected `rule`, got {}", tail.fmt(output).await),
[Pos::Range(output.range.clone()).into()],
));
continue;
};
let arrow = tail.i().i("=>").await;
let (pat, body) = match tail.split_once(|t| t.is_kw(arrow.clone())) {
Some((a, b)) => (a, b),
None => {
errors.extend(mk_errv(
tail.i().i("no => in macro rule").await,
"The pattern and body of a rule must be separated by a =>",
[Pos::Range(tail.pos()).into()],
));
continue;
},
};
rules.push(Rule {
comments: item.output,
pos: Pos::Range(tail.pos()),
pattern: parse_mtree(pat, path.clone()).await?,
kind: RuleKind::Native(Code::from_code(
CodeLocator::to_rule(tail.i().i(&path.unreverse()).await, macro_i, i as u16),
parse_mtree(body, path.clone()).await?,
)),
})
let (function, value) = tail.split_at(last_idx as u32);
let pos = tail.sr().pos();
if !function.iter().all(TokTree::is_fluff) {
let (f_psb, x_psb) = psb.split();
let x_expr = parse_expr(ctx, path.clone(), x_psb, value).boxed_local().await?;
let f_expr = parse_expr(ctx, path, f_psb, function).boxed_local().await?;
return Ok(ExprKind::Call(f_expr, x_expr).at(pos));
}
let Parsed { output: head, tail } = try_pop_no_fluff(ctx, value).await?;
match &head.tok {
Token::BR | Token::Comment(_) => panic!("Fluff skipped"),
Token::Bottom(b) => Ok(ExprKind::Bottom(b.clone()).at(pos.clone())),
Token::Handle(expr) => Ok(expr.clone()),
Token::NS(n, nametail) => {
let mut nametail = nametail;
let mut segments = vec![n.clone()];
while let Token::NS(n, newtail) = &nametail.tok {
segments.push(n.clone());
nametail = newtail;
}
let Token::Name(n) = &nametail.tok else {
return Err(mk_errv(
ctx.i().i("Loose namespace prefix in constant").await,
"Namespace prefixes in constants must be followed by names",
[pos],
));
};
segments.push(n.clone());
Ok(ExprKind::Const(Sym::new(segments, ctx.i()).await.unwrap()).at(pos.clone()))
},
Token::LambdaHead(h) => {
let [TokTree { tok: Token::Name(arg), .. }] = &h[..] else {
return Err(mk_errv(
ctx.i().i("Complex lambda binding in constant").await,
"Lambda args in constants must be identified by a single name",
[pos],
));
};
let lambda_builder = psb.lambda(arg);
let body = parse_expr(ctx, path.clone(), lambda_builder.stack(), tail).boxed_local().await?;
Ok(ExprKind::Lambda(lambda_builder.collect(), body).at(pos.clone()))
},
Token::S(Paren::Round, body) =>
parse_expr(ctx, path, psb, Snippet::new(head, body)).boxed_local().await,
Token::S(..) =>
return Err(mk_errv(
ctx.i().i("Constants may only contain (), not [] or {}").await,
"It seems like you are trying to call a macro. Consider a 'let' line",
[pos],
)),
Token::Name(n) =>
if psb.register_arg(n) {
Ok(ExprKind::Arg.at(pos))
} else {
Ok(ExprKind::Const(Sym::new([n.clone()], ctx.i()).await.unwrap()).at(pos))
},
Token::NewExpr(ex) => Ok(ex.clone()),
}
if let Ok(e) = OrcErrv::new(errors) { Err(e) } else { Ok(rules) }
}

187
orchid-host/src/parsed.rs Normal file
View File

@@ -0,0 +1,187 @@
use std::fmt::Debug;
use std::rc::Rc;
use futures::FutureExt;
use futures::future::join_all;
use hashbrown::HashSet;
use itertools::Itertools;
use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
use orchid_base::interner::Tok;
use orchid_base::location::SrcRange;
use orchid_base::parse::{Comment, Import};
use orchid_base::tl_cache;
use orchid_base::tree::{TokTree, Token};
use crate::api;
use crate::dealias::{ChildErrorKind, ChildResult, Tree};
use crate::expr::Expr;
use crate::system::System;
pub type ParsTokTree = TokTree<Expr, Expr>;
pub type ParsTok = Token<Expr, Expr>;
#[derive(Debug)]
pub struct Item {
pub sr: SrcRange,
pub comments: Vec<Comment>,
pub kind: ItemKind,
}
#[derive(Debug)]
pub enum ItemKind {
Member(ParsedMember),
Import(Import),
}
impl ItemKind {
#[must_use]
pub fn at(self, sr: SrcRange) -> Item { Item { comments: vec![], sr, kind: self } }
}
impl Format for Item {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
let comment_text = self.comments.iter().join("\n");
let item_text = match &self.kind {
ItemKind::Import(i) => format!("import {i}").into(),
ItemKind::Member(mem) => match &mem.kind {
ParsedMemberKind::ParsedConst(expr) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} = {1l}")))
.units([mem.name.rc().into(), expr.print(c).await]),
ParsedMemberKind::DeferredConst(_, sys) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
.units([mem.name.rc().into(), sys.print(c).await]),
ParsedMemberKind::Mod(module) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
},
};
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
.units([comment_text.into(), item_text])
}
}
pub struct ParsedMember {
pub name: Tok<String>,
pub exported: bool,
pub kind: ParsedMemberKind,
}
impl ParsedMember {
#[must_use]
pub fn name(&self) -> Tok<String> { self.name.clone() }
}
impl Debug for ParsedMember {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Member")
.field("name", &self.name)
.field("kind", &self.kind)
.finish_non_exhaustive()
}
}
#[derive(Debug)]
pub enum ParsedMemberKind {
DeferredConst(api::ParsedConstId, System),
ParsedConst(Expr),
Mod(ParsedModule),
}
// TODO: cannot determine alias origin at this stage; parsed tree is never
// walkable!
#[derive(Debug, Default)]
pub struct ParsedModule {
pub exports: Vec<Tok<String>>,
pub items: Vec<Item>,
}
impl ParsedModule {
#[must_use]
pub fn new(items: impl IntoIterator<Item = Item>) -> Self {
let items = items.into_iter().collect_vec();
let exports = (items.iter())
.filter_map(|i| if let ItemKind::Member(m) = &i.kind { Some(m) } else { None })
.filter(|m| m.exported)
.map(|m| m.name.clone())
.collect_vec();
Self { exports, items }
}
pub fn merge(&mut self, other: ParsedModule) {
let mut swap = ParsedModule::default();
std::mem::swap(self, &mut swap);
*self = ParsedModule::new(swap.items.into_iter().chain(other.items))
}
#[must_use]
pub fn get_imports(&self) -> impl IntoIterator<Item = &Import> {
(self.items.iter())
.filter_map(|it| if let ItemKind::Import(i) = &it.kind { Some(i) } else { None })
}
pub fn default_item(self, name: Tok<String>, sr: SrcRange) -> Item {
let mem = ParsedMember { exported: true, name, kind: ParsedMemberKind::Mod(self) };
Item { comments: vec![], sr, kind: ItemKind::Member(mem) }
}
}
impl Tree for ParsedModule {
type Ctx<'a> = ();
async fn child(
&self,
key: Tok<String>,
public_only: bool,
(): &mut Self::Ctx<'_>,
) -> ChildResult<'_, Self> {
if public_only && !self.exports.contains(&key) {
return ChildResult::Err(ChildErrorKind::Private);
}
if let Some(member) = (self.items.iter())
.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
.find(|m| m.name == key)
{
match &member.kind {
ParsedMemberKind::DeferredConst(..) | ParsedMemberKind::ParsedConst(_) =>
return ChildResult::Err(ChildErrorKind::Constant),
ParsedMemberKind::Mod(m) => return ChildResult::Ok(m),
}
}
ChildResult::Err(ChildErrorKind::Missing)
}
fn children(&self, public_only: bool) -> HashSet<Tok<String>> {
let mut public: HashSet<_> = self.exports.iter().cloned().collect();
if !public_only {
public.extend(
(self.items.iter())
.filter_map(
|it| if let ItemKind::Member(mem) = &it.kind { Some(&mem.name) } else { None },
)
.cloned(),
)
}
public
}
}
impl Format for ParsedModule {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
let head_str = format!("export ::({})\n", self.exports.iter().join(", "));
Variants::sequence(self.items.len() + 1, "\n", None).units(
[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
)
}
}
/// TODO:
///
/// idea, does the host need an IR here or can we figure out a way to transcribe
/// these? Should we spin off a new stage for value parsing so that ParsTokTree
/// doesn't appear in the interpreter's ingress?
pub struct Const {
pub source: Option<Vec<ParsTokTree>>,
}
/// Selects a code element
///
/// Either the steps point to a constant and rule_loc is None, or the steps
/// point to a module and rule_loc selects a macro rule within that module
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct ConstPath {
steps: Tok<Vec<Tok<String>>>,
}
impl ConstPath {
#[must_use]
pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps } }
}

64
orchid-host/src/subprocess.rs
View File

@@ -1,12 +1,11 @@
use std::cell::RefCell;
use std::path::PathBuf;
use std::io::Write;
use std::pin::Pin;
use std::thread;
use async_process::{self, Child, ChildStdin, ChildStdout};
use async_std::io::{self, BufReadExt, BufReader};
use async_std::sync::Mutex;
use futures::FutureExt;
use futures::AsyncWriteExt;
use futures::future::LocalBoxFuture;
use orchid_api_traits::{Decode, Encode};
use orchid_base::builtin::{ExtInit, ExtPort};
@@ -22,8 +21,6 @@ pub async fn ext_command(
msg_logs: Logger,
ctx: Ctx,
) -> io::Result<ExtInit> {
let prog_pbuf = PathBuf::from(cmd.get_program());
let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy().to_string();
let mut child = async_process::Command::from(cmd)
.stdin(async_process::Stdio::piped())
.stdout(async_process::Stdio::piped())
@@ -36,57 +33,68 @@ pub async fn ext_command(
let mut stdout = child.stdout.take().unwrap();
let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await;
let child_stderr = child.stderr.take().unwrap();
thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || {
async_std::task::block_on(async move {
let mut reader = BufReader::new(child_stderr);
loop {
let mut buf = String::new();
if 0 == reader.read_line(&mut buf).await.unwrap() {
break;
}
logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
(ctx.spawn)(Box::pin(async move {
let mut reader = BufReader::new(child_stderr);
loop {
let mut buf = String::new();
if 0 == reader.read_line(&mut buf).await.unwrap() {
break;
}
})
})?;
logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
}
}));
Ok(ExtInit {
header,
port: Box::new(Subprocess {
name: header.name.clone(),
child: RefCell::new(Some(child)),
stdin: Mutex::new(Box::pin(stdin)),
stdin: Some(Mutex::new(Box::pin(stdin))),
stdout: Mutex::new(Box::pin(stdout)),
ctx,
}),
header,
})
}
pub struct Subprocess {
name: String,
child: RefCell<Option<Child>>,
stdin: Mutex<Pin<Box<ChildStdin>>>,
stdin: Option<Mutex<Pin<Box<ChildStdin>>>>,
stdout: Mutex<Pin<Box<ChildStdout>>>,
ctx: Ctx,
}
impl Drop for Subprocess {
fn drop(&mut self) {
let mut child = self.child.borrow_mut().take().unwrap();
let name = self.name.clone();
if std::thread::panicking() {
eprintln!("Killing extension {name}");
// we don't really care to handle errors here
let _: Result<_, _> = std::io::stderr().flush();
let _: Result<_, _> = child.kill();
return;
}
let stdin = self.stdin.take().unwrap();
(self.ctx.spawn)(Box::pin(async move {
let status = child.status().await.expect("Extension exited with error");
assert!(status.success(), "Extension exited with error {status}");
stdin.lock().await.close().await.unwrap();
let status = (child.status().await)
.unwrap_or_else(|e| panic!("{e}, extension {name} exited with error"));
assert!(status.success(), "Extension {name} exited with error {status}");
}))
}
}
impl ExtPort for Subprocess {
fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
async { send_msg(Pin::new(&mut *self.stdin.lock().await), msg).await.unwrap() }.boxed_local()
Box::pin(async {
send_msg(Pin::new(&mut *self.stdin.as_ref().unwrap().lock().await), msg).await.unwrap()
})
}
fn recv<'a>(
&'a self,
cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
) -> LocalBoxFuture<'a, ()> {
fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
Box::pin(async {
std::io::Write::flush(&mut std::io::stderr()).unwrap();
match recv_msg(self.stdout.lock().await.as_mut()).await {
Ok(msg) => cb(&msg).await,
Err(e) if e.kind() == io::ErrorKind::BrokenPipe => (),
Ok(msg) => Some(msg),
Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => None,
Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
}
})

178
orchid-host/src/system.rs
View File

@@ -1,20 +1,20 @@
use std::collections::VecDeque;
use std::fmt;
use std::future::Future;
use std::rc::{Rc, Weak};
use std::{fmt, mem};
use async_stream::stream;
use derive_destructure::destructure;
use futures::StreamExt;
use futures::FutureExt;
use futures::future::join_all;
use hashbrown::HashMap;
use itertools::Itertools;
use memo_map::MemoMap;
use orchid_base::char_filter::char_filter_match;
use orchid_base::clone;
use orchid_base::error::{OrcErrv, OrcRes};
use orchid_base::format::{FmtCtx, FmtUnit, Format};
use orchid_base::interner::Tok;
use orchid_base::location::Pos;
use orchid_base::interner::{Interner, Tok};
use orchid_base::location::SrcRange;
use orchid_base::name::{NameLike, Sym, VName};
use orchid_base::parse::Comment;
use orchid_base::reqnot::{ReqNot, Requester};
use orchid_base::tree::ttv_from_api;
@@ -23,17 +23,23 @@ use substack::{Stackframe, Substack};
use crate::api;
use crate::ctx::Ctx;
use crate::dealias::{absolute_path, walk};
use crate::expr::{ExprParseCtx, ExprWillPanic};
use crate::expr_store::ExprStore;
use crate::extension::{Extension, WeakExtension};
use crate::tree::{ItemKind, Member, Module, ParsTokTree};
use crate::parsed::{Item, ItemKind, ParsTokTree, ParsedMember, ParsedMemberKind, ParsedModule};
use crate::tree::Root;
#[derive(destructure)]
struct SystemInstData {
deps: Vec<System>,
ctx: Ctx,
ext: Extension,
decl_id: api::SysDeclId,
lex_filter: api::CharFilter,
id: api::SysId,
line_types: Vec<Tok<String>>,
pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
}
impl Drop for SystemInstData {
fn drop(&mut self) { self.ext.system_drop(self.id); }
@@ -52,41 +58,127 @@ impl fmt::Debug for SystemInstData {
#[derive(Clone, Debug)]
pub struct System(Rc<SystemInstData>);
impl System {
#[must_use]
pub fn id(&self) -> api::SysId { self.0.id }
#[must_use]
pub fn ext(&self) -> &Extension { &self.0.ext }
#[must_use]
pub fn ctx(&self) -> &Ctx { &self.0.ctx }
#[must_use]
pub fn i(&self) -> &Interner { &self.0.ctx.i }
#[must_use]
pub fn deps(&self) -> &[System] { &self.0.deps }
#[must_use]
pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { self.0.ext.reqnot() }
#[must_use]
pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
self.reqnot().request(api::GetMember(self.0.id, id)).await
}
#[must_use]
pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
#[must_use]
pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
/// Have this system lex a part of the source. It is assumed that
/// [Self::can_lex] was called and returned true.
pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
&self,
source: Tok<String>,
src: Sym,
pos: u32,
r: impl FnMut(u32) -> F,
) -> api::OrcResult<Option<api::LexedExpr>> {
self.0.ext.lex_req(source, pos, self.id(), r).await
self.0.ext.lex_req(source, src, pos, self.id(), r).await
}
#[must_use]
pub fn can_parse(&self, ltyp: Tok<String>) -> bool { self.0.line_types.contains(&ltyp) }
pub fn line_types(&self) -> impl Iterator<Item = &Tok<String>> + '_ { self.0.line_types.iter() }
pub async fn parse(
&self,
path: Substack<'_, Tok<String>>,
line: Vec<ParsTokTree>,
exported: bool,
comments: Vec<Comment>,
) -> OrcRes<Vec<ParsTokTree>> {
let line =
join_all(line.iter().map(|t| async { t.to_api(&mut |n, _| match *n {}).await })).await;
callback: &mut impl AsyncFnMut(Substack<'_, Tok<String>>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
) -> OrcRes<Vec<Item>> {
let src_path = line.first().expect("cannot be empty").sr.path();
let line = join_all(line.into_iter().map(|t| async {
let mut expr_store = self.0.ext.exprs().clone();
t.into_api(&mut expr_store, &mut ExprWillPanic).await
}))
.await;
let comments = comments.iter().map(Comment::to_api).collect_vec();
match self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line }).await {
Ok(parsed) => Ok(ttv_from_api(parsed, &mut self.ctx().clone(), &self.ctx().i).await),
let req = api::ParseLine {
module: self.i().i(&path.unreverse()).await.to_api(),
src: src_path.to_api(),
exported,
sys: self.id(),
comments,
line,
};
match self.reqnot().request(req).await {
Ok(parsed_v) => {
let mut ext_exprs = self.ext().exprs().clone();
struct ConvCtx<'a> {
sys: &'a System,
src_path: &'a Sym,
i: &'a Interner,
ext_exprs: &'a mut ExprStore,
pctx: &'a mut ExprParseCtx<'a>,
}
async fn conv(
parsed_v: Vec<api::ParsedLine>,
module: Substack<'_, Tok<String>>,
callback: &'_ mut impl AsyncFnMut(
Substack<'_, Tok<String>>,
Vec<ParsTokTree>,
) -> OrcRes<Vec<Item>>,
ctx: &mut ConvCtx<'_>,
) -> OrcRes<Vec<Item>> {
let mut items = Vec::new();
for parsed in parsed_v {
let (name, exported, kind) = match parsed.kind {
api::ParsedLineKind::Member(api::ParsedMember { name, exported, kind }) =>
(name, exported, kind),
api::ParsedLineKind::Recursive(rec) => {
let tokens = ttv_from_api(rec, ctx.ext_exprs, ctx.pctx, ctx.src_path, ctx.i).await;
items.extend(callback(module.clone(), tokens).await?);
continue;
},
};
let name = ctx.i.ex(name).await;
let mkind = match kind {
api::ParsedMemberKind::Module(items) => {
let items =
conv(items, module.push(name.clone()), callback, ctx).boxed_local().await?;
ParsedMemberKind::Mod(ParsedModule::new(items))
},
api::ParsedMemberKind::Constant(cid) =>
ParsedMemberKind::DeferredConst(cid, ctx.sys.clone()),
};
items.push(Item {
comments: join_all(
parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone(), ctx.i)),
)
.await,
sr: SrcRange::from_api(&parsed.source_range, ctx.i).await,
kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
})
}
Ok(items)
}
conv(parsed_v, path, callback, &mut ConvCtx {
i: self.i(),
ext_exprs: &mut ext_exprs,
pctx: &mut ExprParseCtx { ctx: self.ctx(), exprs: self.ext().exprs() },
src_path: &src_path,
sys: self,
})
.await
},
Err(e) => Err(OrcErrv::from_api(&e, &self.ctx().i).await),
}
}
#[must_use]
pub async fn request(&self, req: Vec<u8>) -> Vec<u8> {
self.reqnot().request(api::SysFwded(self.id(), req)).await
}
@@ -96,7 +188,24 @@ impl System {
this.ctx.owned_atoms.write().await.remove(&drop);
}))
}
#[must_use]
pub fn downgrade(&self) -> WeakSystem { WeakSystem(Rc::downgrade(&self.0)) }
/// Implementation of [api::ResolveNames]
pub(crate) async fn name_resolver(
&self,
orig: api::ParsedConstId,
) -> impl AsyncFnMut(&[Tok<String>]) -> Option<VName> + use<> {
let root = self.0.ctx.root.read().await.upgrade().expect("find_names when root not in context");
let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
let ctx = self.0.ctx.clone();
async move |rel| {
let cwd = orig.split_last().1;
let abs = absolute_path(cwd, rel, &ctx.i).await.ok()?;
let root_data = &mut *root.0.write().await;
let walk_ctx = &mut (ctx.clone(), &mut root_data.consts);
walk(&root_data.root, false, abs.iter(), walk_ctx).await.is_ok().then_some(abs)
}
}
}
impl Format for System {
async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -108,6 +217,7 @@ impl Format for System {
pub struct WeakSystem(Weak<SystemInstData>);
impl WeakSystem {
#[must_use]
pub fn upgrade(&self) -> Option<System> { self.0.upgrade().map(System) }
}
@@ -116,19 +226,25 @@ pub struct SystemCtor {
pub(crate) ext: WeakExtension,
}
impl SystemCtor {
#[must_use]
pub fn name(&self) -> &str { &self.decl.name }
#[must_use]
pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
#[must_use]
pub fn depends(&self) -> impl ExactSizeIterator<Item = &str> {
self.decl.depends.iter().map(|s| &**s)
}
#[must_use]
pub fn id(&self) -> api::SysDeclId { self.decl.id }
pub async fn run<'a>(&self, depends: impl IntoIterator<Item = &'a System>) -> System {
let depends = depends.into_iter().map(|si| si.id()).collect_vec();
#[must_use]
pub async fn run(&self, deps: Vec<System>) -> (Root, System) {
let depends = deps.iter().map(|si| si.id()).collect_vec();
debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
let id = ext.ctx().next_sys_id();
let sys_inst = ext.reqnot().request(api::NewSystem { depends, id, system: self.decl.id }).await;
let data = System(Rc::new(SystemInstData {
deps,
decl_id: self.decl.id,
ext: ext.clone(),
ctx: ext.ctx().clone(),
@@ -136,24 +252,16 @@ impl SystemCtor {
line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
.await,
id,
const_paths: MemoMap::new(),
}));
let const_root = clone!(data, ext; stream! {
for (k, v) in sys_inst.const_root {
yield Member::from_api(
api::Member { name: k, kind: v },
&mut vec![Tok::from_api(k, &ext.ctx().i).await],
&data,
).await;
}
})
.map(|mem| ItemKind::Member(mem).at(Pos::None))
.collect::<Vec<_>>()
.await;
let api_module_root = api::Module {
members: (sys_inst.const_root.into_iter())
.map(|(k, v)| api::Member { name: k, kind: v, comments: vec![], exported: true })
.collect_vec(),
};
let root = Root::from_api(api_module_root, &data).await;
ext.ctx().systems.write().await.insert(id, data.downgrade());
let mut swap = Module::default();
mem::swap(&mut swap, &mut *ext.ctx().root.write().await);
*ext.ctx().root.write().await = Module::new(swap.items.into_iter().chain(const_root));
data
(root, data)
}
}
@@ -166,7 +274,7 @@ pub enum SysResolvErr {
pub async fn init_systems(
tgts: &[String],
exts: &[Extension],
) -> Result<Vec<System>, SysResolvErr> {
) -> Result<(Root, Vec<System>), SysResolvErr> {
let mut to_load = HashMap::<&str, &SystemCtor>::new();
let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque<&str>>();
while let Some(target) = to_find.pop_front() {
@@ -205,9 +313,11 @@ pub async fn init_systems(
walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
}
let mut systems = HashMap::<&str, System>::new();
let mut root = Root::new(exts.first().unwrap().ctx().clone());
for ctor in to_load_ordered.iter() {
let sys = ctor.run(ctor.depends().map(|n| &systems[n])).await;
let (sys_root, sys) = ctor.run(ctor.depends().map(|n| systems[n].clone()).collect()).await;
systems.insert(ctor.name(), sys);
root = root.merge(&sys_root).await.expect("Conflicting roots");
}
Ok(systems.into_values().collect_vec())
Ok((root, systems.into_values().collect_vec()))
}

725
orchid-host/src/tree.rs
View File

@@ -1,342 +1,459 @@
use std::fmt::Debug;
use std::rc::Rc;
//! This tree isn't Clone because lazy subtrees are guaranteed to only be loaded
//! once
use std::cell::RefCell;
use std::rc::{Rc, Weak};
use async_once_cell::OnceCell;
use async_std::sync::Mutex;
use async_stream::stream;
use futures::future::join_all;
use futures::{FutureExt, StreamExt};
use async_std::sync::RwLock;
use futures::{FutureExt, StreamExt, stream};
use hashbrown::HashMap;
use hashbrown::hash_map::Entry;
use itertools::Itertools;
use never::Never;
use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
use orchid_api::FetchParsedConst;
use orchid_base::clone;
use orchid_base::error::{OrcRes, Reporter, mk_err, mk_errv};
use orchid_base::interner::Tok;
use orchid_base::location::Pos;
use orchid_base::macros::{mtreev_fmt, mtreev_from_api};
use orchid_base::name::Sym;
use orchid_base::parse::{Comment, Import};
use orchid_base::tree::{AtomRepr, TokTree, Token};
use orchid_base::{clone, tl_cache};
use ordered_float::NotNan;
use substack::Substack;
use orchid_base::location::{Pos, SrcRange};
use orchid_base::name::{Sym, VPath};
use orchid_base::reqnot::Requester;
use crate::api;
use crate::atom::AtomHand;
use crate::ctx::Ctx;
use crate::expr::{Expr, mtreev_to_expr};
use crate::macros::{MacTok, MacTree};
use crate::dealias::{ChildErrorKind, Tree, absolute_path, resolv_glob, walk};
use crate::expr::{Expr, ExprParseCtx, PathSetBuilder};
use crate::parsed::{ItemKind, ParsedMemberKind, ParsedModule};
use crate::system::System;
pub type ParsTokTree = TokTree<'static, AtomHand, Never>;
pub type ParsTok = Token<'static, AtomHand, Never>;
#[derive(Debug)]
pub struct Item {
pub pos: Pos,
pub comments: Vec<Comment>,
pub kind: ItemKind,
pub struct RootData {
pub root: Module,
pub consts: HashMap<Sym, Expr>,
pub ctx: Ctx,
}
#[derive(Debug)]
pub enum ItemKind {
Member(Member),
Export(Tok<String>),
Import(Import),
Macro(Option<NotNan<f64>>, Vec<Rule>),
}
impl ItemKind {
pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } }
}
impl Item {
pub async fn from_api(tree: api::Item, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
let kind = match tree.kind {
api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys).await),
api::ItemKind::Import(name) => ItemKind::Import(Import {
path: Sym::from_api(name, &sys.ctx().i).await.iter().cloned().collect(),
name: None,
}),
api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &sys.ctx().i).await),
api::ItemKind::Macro(macro_block) => {
let mut rules = Vec::new();
for rule in macro_block.rules {
let mut comments = Vec::new();
for comment in rule.comments {
comments.push(Comment::from_api(&comment, &sys.ctx().i).await);
}
let pos = Pos::from_api(&rule.location, &sys.ctx().i).await;
let pattern = mtreev_from_api(&rule.pattern, &sys.ctx().i, &mut {
clone!(pos, sys);
move |a| {
clone!(pos, sys);
Box::pin(async move {
MacTok::Atom(AtomHand::from_api(a, pos.clone(), &mut sys.ctx().clone()).await)
})
}
})
.await;
rules.push(Rule { pos, pattern, kind: RuleKind::Remote(sys.clone(), rule.id), comments });
}
ItemKind::Macro(macro_block.priority, rules)
},
#[derive(Clone)]
pub struct Root(pub Rc<RwLock<RootData>>);
impl Root {
#[must_use]
pub fn new(ctx: Ctx) -> Self {
Root(Rc::new(RwLock::new(RootData {
root: Module::default(),
consts: HashMap::default(),
ctx,
})))
}
#[must_use]
pub async fn from_api(api: api::Module, sys: &System) -> Self {
let mut consts = HashMap::new();
let mut tfac = TreeFromApiCtx { consts: &mut consts, path: sys.i().i(&[][..]).await, sys };
let root = Module::from_api(api, &mut tfac).await;
Root(Rc::new(RwLock::new(RootData { root, consts, ctx: sys.ctx().clone() })))
}
pub async fn merge(&self, new: &Root) -> Result<Self, MergeErr> {
let this = self.0.read().await;
let that = new.0.read().await;
let mut consts =
this.consts.iter().chain(&that.consts).map(|(k, v)| (k.clone(), v.clone())).collect();
let root = this.root.merge(&that.root, this.ctx.clone(), &mut consts).await?;
Ok(Self(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() }))))
}
#[must_use]
pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym, rep: &Reporter) -> Self {
let mut ref_this = self.0.write().await;
let this = &mut *ref_this;
let mut deferred_consts = HashMap::new();
let mut tfpctx = FromParsedCtx {
pars_root: parsed,
deferred_consts: &mut deferred_consts,
pars_prefix: pars_prefix.clone(),
consts: &mut this.consts,
root: &this.root,
ctx: &this.ctx,
rep,
};
let mut comments = Vec::new();
for comment in tree.comments.iter() {
comments.push(Comment::from_api(comment, &sys.ctx().i).await)
let mut module = Module::from_parsed(parsed, pars_prefix.clone(), &mut tfpctx).await;
for step in pars_prefix.iter().rev() {
let kind = OnceCell::from(MemberKind::Module(module));
let members = HashMap::from([(
step.clone(),
Rc::new(Member { public: true, lazy: RefCell::new(None), kind }),
)]);
module = Module { imports: HashMap::new(), members }
}
Self { pos: Pos::from_api(&tree.location, &sys.ctx().i).await, comments, kind }
let mut consts = this.consts.clone();
let root = (this.root.merge(&module, this.ctx.clone(), &mut consts).await)
.expect("Merge conflict between parsed and existing module");
let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
*this.ctx.root.write().await = new.downgrade();
for (path, (sys_id, pc_id)) in deferred_consts {
let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
let api_expr = sys.reqnot().request(FetchParsedConst { id: pc_id, sys: sys.id() }).await;
let mut xp_ctx = ExprParseCtx { ctx: &this.ctx, exprs: sys.ext().exprs() };
let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), &mut xp_ctx).await;
new.0.write().await.consts.insert(path, expr);
}
new
}
pub async fn get_const_value(&self, name: Sym, pos: Pos) -> OrcRes<Expr> {
let this = &mut *self.0.write().await;
// shortcut for previously visited
if let Some(val) = this.consts.get(&name) {
return Ok(val.clone());
}
// load the node, then check if this "walk" call added it to the map
let ctx = this.ctx.clone();
let module =
walk(&this.root, false, name.iter().cloned(), &mut (ctx.clone(), &mut this.consts)).await;
if let Some(val) = this.consts.get(&name) {
return Ok(val.clone());
}
match module {
Ok(_) => Err(mk_errv(
ctx.i.i("module used as constant").await,
format!("{name} is a module, not a constant"),
[pos],
)),
Err(e) => match e.kind {
ChildErrorKind::Private => panic!("public_only is false"),
ChildErrorKind::Constant => panic!("Tree refers to constant not in table"),
ChildErrorKind::Missing => Err(mk_errv(
ctx.i.i("Constant does not exist").await,
format!("{name} does not refer to a constant"),
[pos],
)),
},
}
}
#[must_use]
pub fn downgrade(&self) -> WeakRoot { WeakRoot(Rc::downgrade(&self.0)) }
}
#[derive(Clone)]
pub struct WeakRoot(Weak<RwLock<RootData>>);
impl WeakRoot {
#[must_use]
pub fn new() -> Self { Self(Weak::new()) }
#[must_use]
pub fn upgrade(&self) -> Option<Root> { Some(Root(self.0.upgrade()?)) }
}
impl Default for WeakRoot {
fn default() -> Self { Self::new() }
}
pub struct TreeFromApiCtx<'a> {
pub sys: &'a System,
pub consts: &'a mut HashMap<Sym, Expr>,
pub path: Tok<Vec<Tok<String>>>,
}
impl<'a> TreeFromApiCtx<'a> {
#[must_use]
pub async fn push<'c>(&'c mut self, name: Tok<String>) -> TreeFromApiCtx<'c> {
let path = self.sys.ctx().i.i(&self.path.iter().cloned().chain([name]).collect_vec()).await;
TreeFromApiCtx { path, consts: &mut *self.consts, sys: self.sys }
}
}
impl Format for Item {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
let comment_text = self.comments.iter().join("\n");
let item_text = match &self.kind {
ItemKind::Import(i) => format!("import {i}").into(),
ItemKind::Export(e) => format!("export {e}").into(),
ItemKind::Macro(None, rules) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("macro {{\n\t{0}\n}}")))
.units([Variants::sequence(rules.len(), "\n", None)
.units(join_all(rules.iter().map(|r| r.print(c))).await)]),
ItemKind::Member(mem) => match mem.kind.get() {
None => format!("lazy {}", mem.name).into(),
Some(MemberKind::Const(val)) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} = {1}")))
.units([mem.name.rc().into(), val.print(c).await]),
Some(MemberKind::Mod(module)) =>
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
},
_ => panic!(),
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ResolvedImport {
target: Sym,
sr: SrcRange,
}
#[derive(Clone, Default)]
pub struct Module {
pub imports: HashMap<Tok<String>, Result<ResolvedImport, Vec<ResolvedImport>>>,
pub members: HashMap<Tok<String>, Rc<Member>>,
}
impl Module {
#[must_use]
pub async fn from_api(api: api::Module, ctx: &mut TreeFromApiCtx<'_>) -> Self {
let mut members = HashMap::new();
for mem in api.members {
let mem_name = ctx.sys.i().ex(mem.name).await;
let vname = VPath::new(ctx.path.iter().cloned()).name_with_suffix(mem_name.clone());
let name = vname.to_sym(ctx.sys.i()).await;
let (lazy, kind) = match mem.kind {
api::MemberKind::Lazy(id) =>
(Some(LazyMemberHandle { id, sys: ctx.sys.id(), path: name.clone() }), None),
api::MemberKind::Const(val) => {
let mut expr_ctx = ExprParseCtx { ctx: ctx.sys.ctx(), exprs: ctx.sys.ext().exprs() };
let expr = Expr::from_api(&val, PathSetBuilder::new(), &mut expr_ctx).await;
ctx.consts.insert(name.clone(), expr);
(None, Some(MemberKind::Const))
},
api::MemberKind::Module(m) => {
let m = Self::from_api(m, &mut ctx.push(mem_name.clone()).await).boxed_local().await;
(None, Some(MemberKind::Module(m)))
},
};
members.insert(
mem_name.clone(),
Rc::new(Member {
public: mem.exported,
lazy: RefCell::new(lazy),
kind: kind.map_or_else(OnceCell::new, OnceCell::from),
}),
);
}
Self { members, imports: HashMap::new() }
}
#[must_use]
async fn from_parsed(parsed: &ParsedModule, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
let imports_by_name = (parsed.get_imports().into_iter())
.filter_map(|i| Some((i.name.clone()?, i)))
.into_group_map();
let mut glob_imports_by_name = HashMap::<_, Vec<_>>::new();
for import in parsed.get_imports().into_iter().filter(|i| i.name.is_none()) {
let pos = import.sr.pos();
match absolute_path(&path, &import.path, &ctx.ctx.i).await {
Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, pos, &import.path.to_string()).await),
Ok(abs_path) => {
let names_res = match abs_path.strip_prefix(&ctx.pars_prefix[..]) {
None => {
let mut tree_ctx = (ctx.ctx.clone(), &mut *ctx.consts);
resolv_glob(&path, ctx.root, &abs_path, pos, &ctx.ctx.i, &mut tree_ctx).await
},
Some(sub_tgt) => {
let sub_path = (path.strip_prefix(&ctx.pars_prefix[..]))
.expect("from_parsed called with path outside pars_prefix");
resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &ctx.ctx.i, &mut ()).await
},
};
let abs_path = abs_path.to_sym(&ctx.ctx.i).await;
match names_res {
Err(e) => ctx.rep.report(e),
Ok(names) =>
for name in names {
match glob_imports_by_name.entry(name) {
Entry::Occupied(mut o) => o.get_mut().push((abs_path.clone(), import.sr.clone())),
Entry::Vacant(v) => {
v.insert_entry(vec![(abs_path.clone(), import.sr.clone())]);
},
}
},
}
},
}
}
let mut imports = HashMap::new();
let conflicting_imports_msg = ctx.ctx.i.i("Conflicting imports").await;
for (key, values) in imports_by_name {
if values.len() == 1 {
let import = values.into_iter().next().unwrap();
let sr = import.sr.clone();
let abs_path_res = absolute_path(&path, &import.clone().mspath(), &ctx.ctx.i).await;
match abs_path_res {
Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, sr.pos(), &import.to_string()).await),
Ok(abs_path) => {
imports
.insert(key, Ok(ResolvedImport { target: abs_path.to_sym(&ctx.ctx.i).await, sr }));
},
}
} else {
for item in values {
ctx.rep.report(mk_err(
conflicting_imports_msg.clone(),
format!("{key} is imported multiple times from different modules"),
[item.sr.pos().into()],
));
}
}
}
for (key, values) in glob_imports_by_name {
if !imports.contains_key(&key) {
let i = &ctx.ctx.i;
let values = stream::iter(values)
.then(|(n, sr)| {
clone!(key; async move {
ResolvedImport { target: n.to_vname().suffix([key.clone()]).to_sym(i).await, sr }
})
})
.collect::<Vec<_>>()
.await;
imports.insert(key, if values.len() == 1 { Ok(values[0].clone()) } else { Err(values) });
}
}
let self_referential_msg = ctx.ctx.i.i("Self-referential import").await;
for (key, value) in imports.iter() {
let Ok(import) = value else { continue };
if import.target.strip_prefix(&path[..]).is_some_and(|t| t.starts_with(&[key.clone()])) {
ctx.rep.report(mk_err(
self_referential_msg.clone(),
format!("import {} points to itself or a path within itself", &import.target),
[import.sr.pos().into()],
));
}
}
let mut members = HashMap::new();
for item in &parsed.items {
match &item.kind {
ItemKind::Member(mem) => {
let path = path.to_vname().suffix([mem.name.clone()]).to_sym(&ctx.ctx.i).await;
let kind = OnceCell::from(MemberKind::from_parsed(&mem.kind, path.clone(), ctx).await);
members.insert(
mem.name.clone(),
Rc::new(Member { kind, lazy: RefCell::default(), public: mem.exported }),
);
},
ItemKind::Import(_) => (),
}
}
Module { imports, members }
}
pub async fn merge(
&self,
other: &Module,
ctx: Ctx,
consts: &mut HashMap<Sym, Expr>,
) -> Result<Module, MergeErr> {
if !self.imports.is_empty() || !other.imports.is_empty() {
return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Imports });
}
let mut members = HashMap::new();
for (key, mem) in &other.members {
let Some(own) = self.members.get(key) else {
members.insert(key.clone(), mem.clone());
continue;
};
if own.public != mem.public {
return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Visibility });
}
match (own.kind(ctx.clone(), consts).await, mem.kind(ctx.clone(), consts).await) {
(MemberKind::Module(own_sub), MemberKind::Module(sub)) => {
match own_sub.merge(sub, ctx.clone(), consts).boxed_local().await {
Ok(module) => {
members.insert(
key.clone(),
Rc::new(Member {
lazy: RefCell::new(None),
public: own.public,
kind: OnceCell::from(MemberKind::Module(module)),
}),
);
},
Err(mut e) => {
e.path = e.path.prefix([key.clone()]);
return Err(e);
},
}
},
_ => return Err(MergeErr { path: VPath::new([key.clone()]), kind: MergeErrKind::Const }),
}
}
for (key, mem) in &self.members {
if let Entry::Vacant(slot) = members.entry(key.clone()) {
slot.insert(mem.clone());
}
}
Ok(Module { imports: HashMap::new(), members })
}
}
#[derive(Debug)]
pub struct MergeErr {
pub path: VPath,
pub kind: MergeErrKind,
}
#[derive(Debug)]
pub enum MergeErrKind {
Imports,
Visibility,
Const,
}
pub struct FromParsedCtx<'a> {
pars_prefix: Sym,
pars_root: &'a ParsedModule,
root: &'a Module,
consts: &'a mut HashMap<Sym, Expr>,
rep: &'a Reporter,
ctx: &'a Ctx,
deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
}
impl Tree for Module {
type Ctx<'a> = (Ctx, &'a mut HashMap<Sym, Expr>);
async fn child(
&self,
key: Tok<String>,
public_only: bool,
(ctx, consts): &mut Self::Ctx<'_>,
) -> crate::dealias::ChildResult<'_, Self> {
let Some(member) = self.members.get(&key) else {
return Err(ChildErrorKind::Missing);
};
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
.units([comment_text.into(), item_text])
if public_only && !member.public {
return Err(ChildErrorKind::Private);
}
match &member.kind(ctx.clone(), consts).await {
MemberKind::Module(m) => Ok(m),
MemberKind::Const => Err(ChildErrorKind::Constant),
}
}
fn children(&self, public_only: bool) -> hashbrown::HashSet<Tok<String>> {
self.members.iter().filter(|(_, v)| !public_only || v.public).map(|(k, _)| k.clone()).collect()
}
}
pub struct Member {
name: Tok<String>,
kind: OnceCell<MemberKind>,
lazy: Mutex<Option<LazyMemberHandle>>,
pub public: bool,
pub lazy: RefCell<Option<LazyMemberHandle>>,
pub kind: OnceCell<MemberKind>,
}
impl Member {
pub fn name(&self) -> Tok<String> { self.name.clone() }
pub async fn kind(&self) -> &MemberKind {
#[must_use]
pub async fn kind<'a>(&'a self, ctx: Ctx, consts: &mut HashMap<Sym, Expr>) -> &'a MemberKind {
(self.kind.get_or_init(async {
let handle = self.lazy.lock().await.take().expect("Neither known nor lazy");
handle.run().await
let handle = self.lazy.borrow_mut().take().expect("If kind is uninit, lazy must be Some");
handle.run(ctx, consts).await
}))
.await
}
pub async fn kind_mut(&mut self) -> &mut MemberKind {
self.kind().await;
self.kind.get_mut().expect("kind() already filled the cell")
}
pub async fn from_api(api: api::Member, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
path.push(Tok::from_api(api.name, &sys.ctx().i).await);
let kind = match api.kind {
api::MemberKind::Lazy(id) => {
let handle = LazyMemberHandle(id, sys.clone(), path.clone());
return handle.into_member(path.pop().unwrap());
},
api::MemberKind::Const(c) => MemberKind::Const(Code::from_expr(
CodeLocator::to_const(sys.ctx().i.i(&*path).await),
Expr::from_api(&c, &mut sys.ext().clone()).await,
)),
api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, path, sys).await),
};
let name = path.pop().unwrap();
Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
}
pub fn new(name: Tok<String>, kind: MemberKind) -> Self {
Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
}
}
impl Debug for Member {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Member")
.field("name", &self.name)
.field("kind", &self.kind)
.finish_non_exhaustive()
}
}
#[derive(Debug)]
pub enum MemberKind {
Const(Code),
Mod(Module),
Const,
Module(Module),
}
#[derive(Debug, Default)]
pub struct Module {
pub imports: Vec<Sym>,
pub exports: Vec<Tok<String>>,
pub items: Vec<Item>,
}
impl Module {
pub fn new(items: impl IntoIterator<Item = Item>) -> Self {
let items = items.into_iter().collect_vec();
let exports = (items.iter())
.filter_map(|i| match &i.kind {
ItemKind::Export(e) => Some(e.clone()),
_ => None,
})
.collect_vec();
Self { imports: vec![], exports, items }
}
pub async fn from_api(m: api::Module, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
Self::new(
stream! { for item in m.items { yield Item::from_api(item, path, sys).boxed_local().await } }
.collect::<Vec<_>>()
.await,
)
}
pub async fn walk(
&self,
allow_private: bool,
path: impl IntoIterator<Item = Tok<String>>,
) -> Result<&Module, WalkError> {
let mut cur = self;
for (pos, step) in path.into_iter().enumerate() {
let Some(member) = (cur.items.iter())
.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
.find(|m| m.name == step)
else {
return Err(WalkError { pos, kind: WalkErrorKind::Missing });
};
if !allow_private && !cur.exports.contains(&step) {
return Err(WalkError { pos, kind: WalkErrorKind::Private });
}
match member.kind().await {
MemberKind::Const(_) => return Err(WalkError { pos, kind: WalkErrorKind::Constant }),
MemberKind::Mod(m) => cur = m,
}
impl MemberKind {
#[must_use]
async fn from_parsed(parsed: &ParsedMemberKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
match parsed {
ParsedMemberKind::ParsedConst(expr) => {
ctx.consts.insert(path, expr.clone());
MemberKind::Const
},
ParsedMemberKind::DeferredConst(id, sys) => {
ctx.deferred_consts.insert(path, (sys.id(), *id));
MemberKind::Const
},
ParsedMemberKind::Mod(m) =>
MemberKind::Module(Module::from_parsed(m, path, ctx).boxed_local().await),
}
Ok(cur)
}
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub enum WalkErrorKind {
Missing,
Private,
Constant,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct WalkError {
pub pos: usize,
pub kind: WalkErrorKind,
}
impl Format for Module {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
let import_str = self.imports.iter().map(|i| format!("import {i}")).join("\n");
let head_str = format!("{import_str}\nexport ::({})\n", self.exports.iter().join(", "));
Variants::sequence(self.items.len() + 1, "\n", None).units(
[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
)
}
}
pub struct LazyMemberHandle(api::TreeId, System, Vec<Tok<String>>);
pub struct LazyMemberHandle {
id: api::TreeId,
sys: api::SysId,
path: Sym,
}
impl LazyMemberHandle {
pub async fn run(self) -> MemberKind {
match self.1.get_tree(self.0).await {
api::MemberKind::Const(c) => MemberKind::Const(Code {
bytecode: Expr::from_api(&c, &mut self.1.ext().clone()).await.into(),
locator: CodeLocator { steps: self.1.ctx().i.i(&self.2).await, rule_loc: None },
source: None,
}),
api::MemberKind::Module(m) =>
MemberKind::Mod(Module::from_api(m, &mut { self.2 }, &self.1).await),
api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run().boxed_local().await,
#[must_use]
pub async fn run(self, ctx: Ctx, consts: &mut HashMap<Sym, Expr>) -> MemberKind {
let sys = ctx.system_inst(self.sys).await.expect("Missing system for lazy member");
match sys.get_tree(self.id).await {
api::MemberKind::Const(c) => {
let mut pctx = ExprParseCtx { ctx: &ctx, exprs: sys.ext().exprs() };
consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await);
MemberKind::Const
},
api::MemberKind::Module(m) => MemberKind::Module(
Module::from_api(m, &mut TreeFromApiCtx { sys: &sys, consts, path: self.path.tok() }).await,
),
api::MemberKind::Lazy(id) => Self { id, ..self }.run(ctx, consts).boxed_local().await,
}
}
pub fn into_member(self, name: Tok<String>) -> Member {
Member { name, kind: OnceCell::new(), lazy: Mutex::new(Some(self)) }
}
}
#[derive(Debug)]
pub struct Rule {
pub pos: Pos,
pub comments: Vec<Comment>,
pub pattern: Vec<MacTree>,
pub kind: RuleKind,
}
impl Format for Rule {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
FmtUnit::new(
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0b}\n{1} => {2b}"))),
[
self.comments.iter().join("\n").into(),
mtreev_fmt(&self.pattern, c).await,
match &self.kind {
RuleKind::Native(code) => code.print(c).await,
RuleKind::Remote(sys, id) => FmtUnit::new(
tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} #{1}"))),
[sys.print(c).await, format!("{id:?}").into()],
),
},
],
)
}
}
#[derive(Debug)]
pub enum RuleKind {
Remote(System, api::MacroId),
Native(Code),
}
#[derive(Debug)]
pub struct Code {
locator: CodeLocator,
source: Option<Vec<MacTree>>,
bytecode: OnceCell<Expr>,
}
impl Code {
pub fn from_expr(locator: CodeLocator, expr: Expr) -> Self {
Self { locator, source: None, bytecode: expr.into() }
}
pub fn from_code(locator: CodeLocator, code: Vec<MacTree>) -> Self {
Self { locator, source: Some(code), bytecode: OnceCell::new() }
}
pub fn source(&self) -> Option<&Vec<MacTree>> { self.source.as_ref() }
pub fn set_source(&mut self, source: Vec<MacTree>) {
self.source = Some(source);
self.bytecode = OnceCell::new();
}
pub async fn get_bytecode(&self, ctx: &Ctx) -> &Expr {
(self.bytecode.get_or_init(async {
let src = self.source.as_ref().expect("no bytecode or source");
mtreev_to_expr(src, Substack::Bottom, ctx).await.at(Pos::None)
}))
.await
}
}
impl Format for Code {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
if let Some(bc) = self.bytecode.get() {
return bc.print(c).await;
}
if let Some(src) = &self.source {
return mtreev_fmt(src, c).await;
}
panic!("Code must be initialized with at least one state")
}
}
/// Selects a code element
///
/// Either the steps point to a constant and rule_loc is None, or the steps
/// point to a module and rule_loc selects a macro rule within that module
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct CodeLocator {
steps: Tok<Vec<Tok<String>>>,
/// Index of a macro block in the module demarked by the steps, and a rule in
/// that macro
rule_loc: Option<(u16, u16)>,
}
impl CodeLocator {
pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps, rule_loc: None } }
pub fn to_rule(steps: Tok<Vec<Tok<String>>>, macro_i: u16, rule_i: u16) -> Self {
Self { steps, rule_loc: Some((macro_i, rule_i)) }
#[must_use]
pub async fn into_member(self, public: bool) -> Member {
Member { public, kind: OnceCell::new(), lazy: RefCell::new(Some(self)) }
}
}

6
orchid-macros/Cargo.toml Normal file
View File

@@ -0,0 +1,6 @@
[package]
name = "orchid-macros"
version = "0.1.0"
edition = "2024"
[dependencies]

3
orchid-macros/src/main.rs Normal file
View File

@@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

14
orchid-std/Cargo.toml
View File

@@ -1,12 +1,14 @@
[package]
name = "orchid-std"
version = "0.1.0"
edition = "2021"
edition = "2024"
[dependencies]
async-once-cell = "0.5.4"
async-std = "1.13.0"
async-stream = "0.3.6"
futures = "0.3.31"
hashbrown = "0.15.2"
itertools = "0.14.0"
never = "0.1.0"
once_cell = "1.20.2"
@@ -14,6 +16,12 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
orchid-base = { version = "0.1.0", path = "../orchid-base" }
orchid-extension = { version = "0.1.0", path = "../orchid-extension" }
ordered-float = "4.6.0"
orchid-extension = { version = "0.1.0", path = "../orchid-extension", features = [
"tokio",
] }
ordered-float = "5.0.0"
rust_decimal = "1.36.0"
tokio = { version = "1.43.0", features = ["full"] }
[dev-dependencies]
test_executors = "0.3.2"

9
orchid-std/src/lib.rs
View File

@@ -1,7 +1,6 @@
mod number;
// mod macros;
mod std;
mod string;
pub use std::StdSystem;
pub use string::str_atom::OrcString;
pub use std::number::num_atom::{Float, HomoArray, Int, Num};
pub use std::std_system::StdSystem;
pub use std::string::str_atom::OrcString;

34
orchid-std/src/macros/macro_system.rs Normal file
View File

@@ -0,0 +1,34 @@
use never::Never;
use orchid_base::reqnot::Receipt;
use orchid_extension::atom::AtomDynfo;
use orchid_extension::entrypoint::ExtReq;
use orchid_extension::fs::DeclFs;
use orchid_extension::lexer::LexerObj;
use orchid_extension::parser::ParserObj;
use orchid_extension::system::{System, SystemCard};
use orchid_extension::system_ctor::SystemCtor;
use orchid_extension::tree::GenMember;
use crate::macros::mactree_lexer::MacTreeLexer;
#[derive(Default)]
pub struct MacroSystem;
impl SystemCtor for MacroSystem {
type Deps = ();
type Instance = Self;
const NAME: &'static str = "macros";
const VERSION: f64 = 0.00_01;
fn inst() -> Option<Self::Instance> { Some(Self) }
}
impl SystemCard for MacroSystem {
type Ctor = Self;
type Req = Never;
fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> { [] }
}
impl System for MacroSystem {
async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
fn vfs() -> orchid_extension::fs::DeclFs { DeclFs::Mod(&[]) }
fn lexers() -> Vec<LexerObj> { vec![&MacTreeLexer] }
fn parsers() -> Vec<ParserObj> { vec![] }
fn env() -> Vec<GenMember> { vec![] }
}

98
orchid-std/src/macros/mactree.rs Normal file
View File

@@ -0,0 +1,98 @@
use std::borrow::Cow;
use std::fmt::Display;
use std::rc::Rc;
use futures::future::join_all;
use orchid_api::Paren;
use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
use orchid_base::interner::Tok;
use orchid_base::location::Pos;
use orchid_base::name::Sym;
use orchid_base::tl_cache;
use orchid_extension::atom::Atomic;
use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
use orchid_extension::expr::Expr;
#[derive(Debug, Clone)]
pub struct MacTree {
pub pos: Pos,
pub tok: Rc<MacTok>,
}
impl MacTree {}
impl Atomic for MacTree {
type Data = ();
type Variant = OwnedVariant;
}
impl OwnedAtom for MacTree {
type Refs = ();
async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
self.tok.print(c).await
}
}
#[derive(Debug, Clone)]
pub enum MacTok {
S(Paren, Vec<MacTree>),
Name(Sym),
/// Only permitted in arguments to `instantiate_tpl`
Slot,
Value(Expr),
Lambda(Vec<MacTree>, Vec<MacTree>),
Ph(Ph),
}
impl Format for MacTok {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
match self {
Self::Value(v) => v.print(c).await,
Self::Lambda(arg, b) => FmtUnit::new(
tl_cache!(Rc<Variants>: Rc::new(Variants::default()
.unbounded("\\{0b}.{1l}")
.bounded("(\\{0b}.{1b})"))),
[mtreev_fmt(arg, c).await, mtreev_fmt(b, c).await],
),
Self::Name(n) => format!("{n}").into(),
Self::Ph(ph) => format!("{ph}").into(),
Self::S(p, body) => FmtUnit::new(
match *p {
Paren::Round => Rc::new(Variants::default().bounded("({0b})")),
Paren::Curly => Rc::new(Variants::default().bounded("{{0b}}")),
Paren::Square => Rc::new(Variants::default().bounded("[{0b}]")),
},
[mtreev_fmt(body, c).await],
),
Self::Slot => "SLOT".into(),
}
}
}
pub async fn mtreev_fmt<'b>(
v: impl IntoIterator<Item = &'b MacTree>,
c: &(impl FmtCtx + ?Sized),
) -> FmtUnit {
FmtUnit::sequence(" ", None, join_all(v.into_iter().map(|t| t.print(c))).await)
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct Ph {
pub name: Tok<String>,
pub kind: PhKind,
}
impl Display for Ph {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.kind {
PhKind::Scalar => write!(f, "${}", self.name),
PhKind::Vector { at_least_one: false, priority: 0 } => write!(f, "..${}", self.name),
PhKind::Vector { at_least_one: true, priority: 0 } => write!(f, "...${}", self.name),
PhKind::Vector { at_least_one: false, priority } => write!(f, "..${}:{priority}", self.name),
PhKind::Vector { at_least_one: true, priority } => write!(f, "...${}:{priority}", self.name),
}
}
}
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
pub enum PhKind {
Scalar,
Vector { at_least_one: bool, priority: u8 },
}

61
orchid-std/src/macros/mactree_lexer.rs Normal file
View File

@@ -0,0 +1,61 @@
use std::ops::RangeInclusive;
use std::rc::Rc;
use futures::FutureExt;
use orchid_base::error::{OrcRes, mk_errv};
use orchid_base::tokens::PARENS;
use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
use orchid_extension::tree::{GenTok, GenTokTree, x_tok};
use crate::macros::mactree::{MacTok, MacTree};
#[derive(Default)]
pub struct MacTreeLexer;
impl Lexer for MacTreeLexer {
const CHAR_FILTER: &'static [RangeInclusive<char>] = &['\''..='\''];
async fn lex<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
let Some(tail2) = tail.strip_prefix('\'') else {
return Err(err_not_applicable(ctx.i()).await.into());
};
let tail3 = tail2.trim_start();
return match mac_tree(tail3, ctx).await {
Ok((tail4, mactree)) => Ok((tail4, x_tok(mactree).at(ctx.pos_tt(tail, tail4)))),
Err(e) => Ok((tail2, GenTok::Bottom(e).at(ctx.pos_lt(1, tail2)))),
};
async fn mac_tree<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, MacTree)> {
for (lp, rp, paren) in PARENS {
let Some(mut body_tail) = tail.strip_prefix(*lp) else { continue };
let mut items = Vec::new();
return loop {
let tail2 = body_tail.trim();
if let Some(tail3) = tail2.strip_prefix(*rp) {
break Ok((tail3, MacTree {
pos: ctx.pos_tt(tail, tail3).pos(),
tok: Rc::new(MacTok::S(*paren, items)),
}));
} else if tail2.is_empty() {
return Err(mk_errv(
ctx.i().i("Unclosed block").await,
format!("Expected closing {rp}"),
[ctx.pos_lt(1, tail)],
));
}
let (new_tail, new_item) = mac_tree(tail2, ctx).boxed_local().await?;
body_tail = new_tail;
items.push(new_item);
};
}
const INTERPOL: &[&str] = &["$", "..$"];
for pref in INTERPOL {
let Some(code) = tail.strip_prefix(pref) else { continue };
todo!("Register parameter, and push this onto the argument stack held in the atom")
}
todo!("recursive lexer call");
return Err(mk_errv(
ctx.i().i("Expected token after '").await,
format!("Expected a token after ', found {tail:?}"),
[ctx.pos_lt(1, tail)],
));
}
}
}

6
orchid-std/src/macros/mod.rs Normal file
View File

@@ -0,0 +1,6 @@
mod macro_system;
pub mod mactree;
mod mactree_lexer;
mod rule;
use mactree::{MacTok, MacTree};

2
orchid-host/src/rule/any_match.rs → orchid-std/src/macros/rule/any_match.rs
View File

@@ -2,9 +2,9 @@ use orchid_base::name::Sym;
use super::scal_match::scalv_match;
use super::shared::AnyMatcher;
use super::state::MatchState;
use super::vec_match::vec_match;
use crate::macros::MacTree;
use crate::rule::state::MatchState;
#[must_use]
pub fn any_match<'a>(

18
orchid-host/src/rule/build.rs → orchid-std/src/macros/rule/build.rs
View File

@@ -1,12 +1,11 @@
use itertools::Itertools;
use orchid_api::PhKind;
use orchid_base::interner::Tok;
use orchid_base::side::Side;
use orchid_base::tree::Ph;
use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
use super::vec_attrs::vec_attrs;
use crate::macros::mactree::{Ph, PhKind};
use crate::macros::{MacTok, MacTree};
use crate::rule::vec_attrs::vec_attrs;
pub type MaxVecSplit<'a> = (&'a [MacTree], (Tok<String>, u8, bool), &'a [MacTree]);
@@ -91,7 +90,6 @@ pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
#[must_use]
fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
match &*pattern.tok {
MacTok::Atom(_) | MacTok::Done(_) => panic!("Atoms and Done aren't supported in matchers"),
MacTok::Name(n) => ScalMatcher::Name(n.clone()),
MacTok::Ph(Ph { name, kind }) => match kind {
PhKind::Vector { .. } => {
@@ -101,7 +99,7 @@ fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
},
MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
MacTok::Lambda(arg, body) => ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
MacTok::Ref(_) | MacTok::Slot(_) => panic!("Extension-only variants"),
MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
}
}
@@ -109,24 +107,22 @@ fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
mod test {
use std::rc::Rc;
use orchid_api::PhKind;
use orchid_base::interner::Interner;
use orchid_base::location::SourceRange;
use orchid_base::location::SrcRange;
use orchid_base::sym;
use orchid_base::tokens::Paren;
use orchid_base::tree::Ph;
use test_executors::spin_on;
use super::mk_any;
use crate::macros::mactree::{Ph, PhKind};
use crate::macros::{MacTok, MacTree};
#[test]
fn test_scan() {
spin_on(async {
let i = Interner::new_master();
let ex = |tok: MacTok| async {
MacTree { tok: Rc::new(tok), pos: SourceRange::mock(&i).await.pos() }
};
let ex =
|tok: MacTok| async { MacTree { tok: Rc::new(tok), pos: SrcRange::mock(&i).await.pos() } };
let pattern = vec![
ex(MacTok::Ph(Ph {
kind: PhKind::Vector { priority: 0, at_least_one: false },

12
orchid-host/src/rule/matcher.rs → orchid-std/src/macros/rule/matcher.rs
View File

@@ -1,20 +1,19 @@
use std::fmt;
use std::rc::Rc;
use itertools::Itertools;
use orchid_api::PhKind;
use orchid_base::interner::Interner;
use orchid_base::location::Pos;
use orchid_base::name::Sym;
use orchid_base::tree::Ph;
use super::any_match::any_match;
use super::build::mk_any;
use super::build::{mk_any, mk_vec};
use super::shared::{AnyMatcher, VecMatcher};
use super::state::{MatchState, StateEntry};
use super::vec_attrs::vec_attrs;
use super::vec_match::vec_match;
use crate::macros::mactree::{Ph, PhKind};
use crate::macros::{MacTok, MacTree};
use crate::rule::build::mk_vec;
pub fn first_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.first().unwrap()).is_some() }
pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwrap()).is_some() }
@@ -30,8 +29,9 @@ impl NamedMatcher {
match last_is_vec(pattern) {
true => Self(mk_any(pattern)),
false => {
let kind: PhKind = PhKind::Vector { priority: 0, at_least_one: false };
let suffix = [MacTok::Ph(Ph { name: i.i("::after").await, kind }).at(Pos::None)];
let kind = PhKind::Vector { priority: 0, at_least_one: false };
let tok = MacTok::Ph(Ph { name: i.i("::after").await, kind });
let suffix = [MacTree { pos: Pos::None, tok: Rc::new(tok) }];
Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
},
}

0
orchid-host/src/rule/mod.rs → orchid-std/src/macros/rule/mod.rs
View File

3
orchid-host/src/rule/scal_match.rs → orchid-std/src/macros/rule/scal_match.rs
View File

@@ -2,8 +2,8 @@ use orchid_base::name::Sym;
use super::any_match::any_match;
use super::shared::ScalMatcher;
use super::state::{MatchState, StateEntry};
use crate::macros::{MacTok, MacTree};
use crate::rule::state::{MatchState, StateEntry};
#[must_use]
pub fn scal_match<'a>(
@@ -16,7 +16,6 @@ pub fn scal_match<'a>(
true => MatchState::from_name(n1.clone(), expr.pos.clone()),
false => MatchState::default(),
}),
(ScalMatcher::Placeh { .. }, MacTok::Done(_)) => None,
(ScalMatcher::Placeh { key }, _) =>
Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
(ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>

0
orchid-host/src/rule/shared.rs → orchid-std/src/macros/rule/shared.rs
View File

4
orchid-host/src/rule/state.rs → orchid-std/src/macros/rule/state.rs
View File

@@ -40,7 +40,9 @@ impl<'a> MatchState<'a> {
pub fn combine(self, s: Self) -> Self {
Self {
placeholders: self.placeholders.into_iter().chain(s.placeholders).collect(),
name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| l.into_iter().chain(r).collect()),
name_posv: join_maps::<_, Vec<Pos>>(self.name_posv, s.name_posv, |_, l, r| {
l.into_iter().chain(r).collect()
}),
}
}
pub fn ph_len(&self, key: &Tok<String>) -> Option<usize> {

3
orchid-host/src/rule/vec_attrs.rs → orchid-std/src/macros/rule/vec_attrs.rs
View File

@@ -1,7 +1,6 @@
use orchid_api::PhKind;
use orchid_base::interner::Tok;
use orchid_base::tree::Ph;
use crate::macros::mactree::{Ph, PhKind};
use crate::macros::{MacTok, MacTree};
/// Returns the name, priority and at_least_one of the expression if it is

2
orchid-host/src/rule/vec_match.rs → orchid-std/src/macros/rule/vec_match.rs
View File

@@ -5,8 +5,8 @@ use orchid_base::name::Sym;
use super::scal_match::scalv_match;
use super::shared::VecMatcher;
use super::state::{MatchState, StateEntry};
use crate::macros::MacTree;
use crate::rule::state::{MatchState, StateEntry};
#[must_use]
pub fn vec_match<'a>(

16
orchid-std/src/main.rs
View File

@@ -1,16 +1,6 @@
use std::mem;
use std::rc::Rc;
use orchid_extension::entrypoint::{ExtensionData, extension_main_logic};
use orchid_extension::entrypoint::ExtensionData;
use orchid_extension::tokio::tokio_main;
use orchid_std::StdSystem;
use tokio::task::{LocalSet, spawn_local};
#[tokio::main(flavor = "current_thread")]
pub async fn main() {
LocalSet::new()
.run_until(async {
let data = ExtensionData::new("orchid-std::main", &[&StdSystem]);
extension_main_logic(data, Rc::new(|fut| mem::drop(spawn_local(fut)))).await;
})
.await
}
pub async fn main() { tokio_main(ExtensionData::new("orchid-std::main", &[&StdSystem])).await }

64
orchid-std/src/number/num_atom.rs
View File

@@ -1,64 +0,0 @@
use orchid_api_derive::Coding;
use orchid_base::error::OrcRes;
use orchid_base::format::FmtUnit;
use orchid_extension::atom::{
AtomFactory, Atomic, AtomicFeatures, MethodSetBuilder, ToAtom, TypAtom,
};
use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
use orchid_extension::conv::TryFromExpr;
use orchid_extension::expr::Expr;
use orchid_extension::system::SysCtx;
use ordered_float::NotNan;
#[derive(Clone, Debug, Coding)]
pub struct Int(pub i64);
impl Atomic for Int {
type Variant = ThinVariant;
type Data = Self;
fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
}
impl ThinAtom for Int {
async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }
}
impl TryFromExpr for Int {
async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
TypAtom::<Int>::try_from_expr(expr).await.map(|t| t.value)
}
}
#[derive(Clone, Debug, Coding)]
pub struct Float(pub NotNan<f64>);
impl Atomic for Float {
type Variant = ThinVariant;
type Data = Self;
fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
}
impl ThinAtom for Float {
async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }
}
impl TryFromExpr for Float {
async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
TypAtom::<Float>::try_from_expr(expr).await.map(|t| t.value)
}
}
pub enum Numeric {
Int(i64),
Float(NotNan<f64>),
}
impl TryFromExpr for Numeric {
async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
match Int::try_from_expr(expr.clone()).await {
Ok(t) => Ok(Numeric::Int(t.0)),
Err(e) => Float::try_from_expr(expr).await.map(|t| Numeric::Float(t.0)).map_err(|e2| e + e2),
}
}
}
impl ToAtom for Numeric {
fn to_atom_factory(self) -> AtomFactory {
match self {
Self::Float(f) => Float(f).factory(),
Self::Int(i) => Int(i).factory(),
}
}
}

27
orchid-std/src/number/num_lexer.rs
View File

@@ -1,27 +0,0 @@
use std::ops::RangeInclusive;
use orchid_base::error::OrcRes;
use orchid_base::number::{Numeric, num_to_err, parse_num};
use orchid_extension::atom::AtomicFeatures;
use orchid_extension::lexer::{LexContext, Lexer};
use orchid_extension::tree::{GenTok, GenTokTree};
use ordered_float::NotNan;
use super::num_atom::{Float, Int};
#[derive(Default)]
pub struct NumLexer;
impl Lexer for NumLexer {
const CHAR_FILTER: &'static [RangeInclusive<char>] = &['0'..='9'];
async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
let ends_at = all.find(|c: char| !c.is_ascii_hexdigit() && !"xX._pP".contains(c));
let (chars, tail) = all.split_at(ends_at.unwrap_or(all.len()));
let fac = match parse_num(chars) {
Ok(Numeric::Float(f)) => Float(f).factory(),
Ok(Numeric::Uint(uint)) => Int(uint.try_into().unwrap()).factory(),
Ok(Numeric::Decimal(dec)) => Float(NotNan::new(dec.try_into().unwrap()).unwrap()).factory(),
Err(e) => return Err(num_to_err(e, ctx.pos(all), ctx.i).await.into()),
};
Ok((tail, GenTok::X(fac).at(ctx.pos(all)..ctx.pos(tail))))
}
}

47
orchid-std/src/std.rs
View File

@@ -1,47 +0,0 @@
use std::rc::Rc;
use never::Never;
use orchid_base::reqnot::Receipt;
use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
use orchid_extension::entrypoint::ExtReq;
use orchid_extension::fs::DeclFs;
use orchid_extension::system::{System, SystemCard};
use orchid_extension::system_ctor::SystemCtor;
use orchid_extension::tree::{MemKind, comments, fun, module, root_mod};
use crate::OrcString;
use crate::number::num_atom::{Float, Int};
use crate::number::num_lexer::NumLexer;
use crate::string::str_atom::{IntStrAtom, StrAtom};
use crate::string::str_lexer::StringLexer;
#[derive(Default)]
pub struct StdSystem;
impl SystemCtor for StdSystem {
type Deps = ();
type Instance = Self;
const NAME: &'static str = "orchid::std";
const VERSION: f64 = 0.00_01;
fn inst() -> Option<Self::Instance> { Some(StdSystem) }
}
impl SystemCard for StdSystem {
type Ctor = Self;
type Req = Never;
fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
[Some(Int::dynfo()), Some(Float::dynfo()), Some(StrAtom::dynfo()), Some(IntStrAtom::dynfo())]
}
}
impl System for StdSystem {
async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer, &NumLexer] }
fn parsers() -> Vec<orchid_extension::parser::ParserObj> { vec![] }
fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
fn env() -> Vec<(String, MemKind)> {
vec![root_mod("std", [], [module(true, "string", [], [comments(
["Concatenate two strings"],
fun(true, "concat", |left: OrcString<'static>, right: OrcString<'static>| async move {
StrAtom::new(Rc::new(left.get_string().await.to_string() + &right.get_string().await))
}),
)])])]
}
}

4
orchid-std/src/std/mod.rs Normal file
View File

@@ -0,0 +1,4 @@
pub mod number;
pub mod string;
pub mod std_system;

1
orchid-std/src/number/mod.rs → orchid-std/src/std/number/mod.rs
View File

@@ -1,2 +1,3 @@
pub mod num_atom;
pub mod num_lexer;
pub mod num_lib;

91
orchid-std/src/std/number/num_atom.rs Normal file
View File

@@ -0,0 +1,91 @@
use orchid_api_derive::Coding;
use orchid_base::error::OrcRes;
use orchid_base::format::FmtUnit;
use orchid_base::number::Numeric;
use orchid_extension::atom::{AtomFactory, Atomic, AtomicFeatures, ToAtom, TypAtom};
use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
use orchid_extension::conv::TryFromExpr;
use orchid_extension::expr::Expr;
use orchid_extension::system::SysCtx;
use ordered_float::NotNan;
use rust_decimal::prelude::Zero;
#[derive(Clone, Debug, Coding)]
pub struct Int(pub i64);
impl Atomic for Int {
type Variant = ThinVariant;
type Data = Self;
}
impl ThinAtom for Int {
async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }
}
impl TryFromExpr for Int {
async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
TypAtom::<Int>::try_from_expr(expr).await.map(|t| t.value)
}
}
#[derive(Clone, Debug, Coding)]
pub struct Float(pub NotNan<f64>);
impl Atomic for Float {
type Variant = ThinVariant;
type Data = Self;
}
impl ThinAtom for Float {
async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }
}
impl TryFromExpr for Float {
async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
Ok(Self(Num::try_from_expr(expr).await?.0.to_f64()))
}
}
pub struct Num(pub Numeric);
impl TryFromExpr for Num {
async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
let e = match Int::try_from_expr(expr.clone()).await {
Ok(t) => return Ok(Num(Numeric::Int(t.0))),
Err(e) => e,
};
match TypAtom::<Float>::try_from_expr(expr).await {
Ok(t) => Ok(Num(Numeric::Float(t.0))),
Err(e2) => Err(e + e2),
}
}
}
impl ToAtom for Num {
fn to_atom_factory(self) -> AtomFactory {
match self.0 {
Numeric::Float(f) => Float(f).factory(),
Numeric::Int(i) => Int(i).factory(),
}
}
}
/// A homogenous fixed length number array that forces all of its elements into
/// the weakest element type. This describes the argument casting behaviour of
/// most numeric operations.
pub enum HomoArray<const N: usize> {
Int([i64; N]),
Float([NotNan<f64>; N]),
}
impl<const N: usize> HomoArray<N> {
pub fn new(n: [Numeric; N]) -> Self {
let mut ints = [0i64; N];
for i in 0..N {
if let Numeric::Int(val) = n[i] {
ints[i] = val
} else {
let mut floats = [NotNan::zero(); N];
for (i, int) in ints.iter().take(i).enumerate() {
floats[i] = NotNan::new(*int as f64).expect("i64 cannot convert to f64 NaN");
}
for j in i..N {
floats[j] = n[j].to_f64();
}
return Self::Float(floats);
}
}
Self::Int(ints)
}
}

24
orchid-std/src/std/number/num_lexer.rs Normal file
View File

@@ -0,0 +1,24 @@
use std::ops::RangeInclusive;
use orchid_base::error::OrcRes;
use orchid_base::number::{num_to_err, parse_num};
use orchid_extension::atom::ToAtom;
use orchid_extension::lexer::{LexContext, Lexer};
use orchid_extension::tree::{GenTokTree, x_tok};
use super::num_atom::Num;
#[derive(Default)]
pub struct NumLexer;
impl Lexer for NumLexer {
const CHAR_FILTER: &'static [RangeInclusive<char>] = &['0'..='9'];
async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
let ends_at = all.find(|c: char| !c.is_ascii_hexdigit() && !"xX._pP".contains(c));
let (chars, tail) = all.split_at(ends_at.unwrap_or(all.len()));
let fac = match parse_num(chars) {
Ok(numeric) => Num(numeric).to_atom_factory(),
Err(e) => return Err(num_to_err(e, ctx.pos(all), &ctx.src, ctx.ctx.i()).await.into()),
};
Ok((tail, x_tok(fac).at(ctx.pos_lt(chars.len(), tail))))
}
}

34
orchid-std/src/std/number/num_lib.rs Normal file
View File

@@ -0,0 +1,34 @@
use orchid_base::number::Numeric;
use orchid_extension::tree::{GenMember, fun, prefix};
use ordered_float::NotNan;
use super::num_atom::{Float, HomoArray, Int, Num};
pub fn gen_num_lib() -> Vec<GenMember> {
prefix("std::number", [
fun(true, "add", |a: Num, b: Num| async move {
Num(match HomoArray::new([a.0, b.0]) {
HomoArray::Int([a, b]) => Numeric::Int(a + b),
HomoArray::Float([a, b]) => Numeric::Float(a + b),
})
}),
fun(true, "neg", |a: Num| async move {
Num(match a.0 {
Numeric::Int(i) => Numeric::Int(-i),
Numeric::Float(f) => Numeric::Float(-f),
})
}),
fun(true, "mul", |a: Num, b: Num| async move {
Num(match HomoArray::new([a.0, b.0]) {
HomoArray::Int([a, b]) => Numeric::Int(a * b),
HomoArray::Float([a, b]) => Numeric::Float(a * b),
})
}),
fun(true, "idiv", |a: Int, b: Int| async move { Int(a.0 / b.0) }),
fun(true, "imod", |a: Int, b: Int| async move { Int(a.0 % b.0) }),
fun(true, "fdiv", |a: Float, b: Float| async move { Float(a.0 / b.0) }),
fun(true, "fmod", |a: Float, b: Float| async move {
Float(a.0 - NotNan::new((a.0 / b.0).trunc()).unwrap() * b.0)
}),
])
}

39
orchid-std/src/std/std_system.rs Normal file
View File

@@ -0,0 +1,39 @@
use never::Never;
use orchid_base::reqnot::Receipt;
use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
use orchid_extension::entrypoint::ExtReq;
use orchid_extension::lexer::LexerObj;
use orchid_extension::parser::ParserObj;
use orchid_extension::system::{System, SystemCard};
use orchid_extension::system_ctor::SystemCtor;
use orchid_extension::tree::{GenMember, merge_trivial};
use super::number::num_lib::gen_num_lib;
use super::string::str_atom::{IntStrAtom, StrAtom};
use super::string::str_lib::gen_str_lib;
use crate::std::number::num_lexer::NumLexer;
use crate::std::string::str_lexer::StringLexer;
use crate::{Float, Int};
#[derive(Default)]
pub struct StdSystem;
impl SystemCtor for StdSystem {
type Deps = ();
type Instance = Self;
const NAME: &'static str = "orchid::std";
const VERSION: f64 = 0.00_01;
fn inst() -> Option<Self::Instance> { Some(Self) }
}
impl SystemCard for StdSystem {
type Ctor = Self;
type Req = Never;
fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
[Some(Int::dynfo()), Some(Float::dynfo()), Some(StrAtom::dynfo()), Some(IntStrAtom::dynfo())]
}
}
impl System for StdSystem {
async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
fn lexers() -> Vec<LexerObj> { vec![&StringLexer, &NumLexer] }
fn parsers() -> Vec<ParserObj> { vec![] }
fn env() -> Vec<GenMember> { merge_trivial([gen_num_lib(), gen_str_lib()]) }
}

1
orchid-std/src/string/mod.rs → orchid-std/src/std/string/mod.rs
View File

@@ -1,2 +1,3 @@
pub mod str_atom;
pub mod str_lexer;
pub mod str_lib;

33
orchid-std/src/string/str_atom.rs → orchid-std/src/std/string/str_atom.rs
View File

@@ -7,7 +7,7 @@ use async_std::io::Write;
use orchid_api_derive::Coding;
use orchid_api_traits::{Encode, Request};
use orchid_base::error::{OrcRes, mk_errv};
use orchid_base::format::FmtUnit;
use orchid_base::format::{FmtCtx, FmtUnit};
use orchid_base::interner::Tok;
use orchid_extension::atom::{AtomMethod, Atomic, MethodSetBuilder, Supports, TypAtom};
use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
@@ -49,7 +49,9 @@ impl OwnedAtom for StrAtom {
async fn serialize(&self, _: SysCtx, sink: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
self.deref().encode(sink).await
}
async fn print(&self, _: SysCtx) -> FmtUnit { format!("{:?}", &*self.0).into() }
async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
format!("{:?}", &*self.0).into()
}
async fn deserialize(mut ctx: impl DeserializeCtx, _: Self::Refs) -> Self {
Self::new(Rc::new(ctx.read::<String>().await))
}
@@ -60,7 +62,6 @@ pub struct IntStrAtom(Tok<String>);
impl Atomic for IntStrAtom {
type Variant = OwnedVariant;
type Data = orchid_api::TStr;
fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
}
impl From<Tok<String>> for IntStrAtom {
fn from(value: Tok<String>) -> Self { Self(value) }
@@ -68,45 +69,47 @@ impl From<Tok<String>> for IntStrAtom {
impl OwnedAtom for IntStrAtom {
type Refs = ();
async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.to_api()) }
async fn print(&self, _ctx: SysCtx) -> FmtUnit { format!("{:?}i", *self.0).into() }
async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
format!("{:?}i", *self.0).into()
}
async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) {
self.0.encode(write).await
}
async fn deserialize(mut ctx: impl DeserializeCtx, _: ()) -> Self {
let s = ctx.decode::<String>().await;
Self(ctx.sys().i.i(&s).await)
Self(ctx.sys().i().i(&s).await)
}
}
#[derive(Clone)]
pub struct OrcString<'a> {
kind: OrcStringKind<'a>,
pub struct OrcString {
kind: OrcStringKind,
ctx: SysCtx,
}
#[derive(Clone)]
pub enum OrcStringKind<'a> {
Val(TypAtom<'a, StrAtom>),
Int(TypAtom<'a, IntStrAtom>),
pub enum OrcStringKind {
Val(TypAtom<StrAtom>),
Int(TypAtom<IntStrAtom>),
}
impl OrcString<'_> {
impl OrcString {
pub async fn get_string(&self) -> Rc<String> {
match &self.kind {
OrcStringKind::Int(tok) => self.ctx.i.ex(**tok).await.rc(),
OrcStringKind::Int(tok) => self.ctx.i().ex(**tok).await.rc(),
OrcStringKind::Val(atom) => atom.request(StringGetVal).await,
}
}
}
impl TryFromExpr for OrcString<'static> {
async fn try_from_expr(expr: Expr) -> OrcRes<OrcString<'static>> {
impl TryFromExpr for OrcString {
async fn try_from_expr(expr: Expr) -> OrcRes<OrcString> {
if let Ok(v) = TypAtom::<StrAtom>::try_from_expr(expr.clone()).await {
return Ok(OrcString { ctx: expr.ctx(), kind: OrcStringKind::Val(v) });
}
let ctx = expr.ctx();
match TypAtom::<IntStrAtom>::try_from_expr(expr).await {
Ok(t) => Ok(OrcString { ctx: t.data.ctx(), kind: OrcStringKind::Int(t) }),
Err(e) => Err(mk_errv(ctx.i.i("A string was expected").await, "", e.pos_iter())),
Err(e) => Err(mk_errv(ctx.i().i("A string was expected").await, "", e.pos_iter())),
}
}
}

40
orchid-std/src/string/str_lexer.rs → orchid-std/src/std/string/str_lexer.rs
View File

@@ -1,12 +1,12 @@
use itertools::Itertools;
use orchid_base::error::{OrcErr, OrcRes, mk_err, mk_errv};
use orchid_base::interner::Interner;
use orchid_base::location::Pos;
use orchid_base::location::SrcRange;
use orchid_base::name::Sym;
use orchid_base::sym;
use orchid_base::tree::wrap_tokv;
use orchid_extension::atom::AtomicFeatures;
use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
use orchid_extension::tree::{GenTok, GenTokTree};
use orchid_extension::tree::{GenTokTree, ref_tok, x_tok};
use super::str_atom::IntStrAtom;
@@ -32,7 +32,7 @@ struct StringError {
impl StringError {
/// Convert into project error for reporting
pub async fn into_proj(self, pos: u32, i: &Interner) -> OrcErr {
pub async fn into_proj(self, path: &Sym, pos: u32, i: &Interner) -> OrcErr {
let start = pos + self.pos;
mk_err(
i.i("Failed to parse string").await,
@@ -41,7 +41,7 @@ impl StringError {
StringErrorKind::BadCodePoint => "The specified number is not a Unicode code point",
StringErrorKind::BadEscSeq => "Unrecognized escape sequence",
},
[Pos::Range(start..start + 1).into()],
[SrcRange::new(start..start + 1, path).pos().into()],
)
}
}
@@ -94,12 +94,10 @@ fn parse_string(str: &str) -> Result<String, StringError> {
#[derive(Default)]
pub struct StringLexer;
impl Lexer for StringLexer {
const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"', '\''..='\''];
async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
let Some((mut tail, delim)) = (all.strip_prefix('"').map(|t| (t, '"')))
.or_else(|| all.strip_prefix('\'').map(|t| (t, '\'')))
else {
return Err(err_not_applicable(ctx.i).await.into());
const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"', '`'..='`'];
async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
let Some(mut tail) = all.strip_prefix('"') else {
return Err(err_not_applicable(ctx.ctx.i()).await.into());
};
let mut ret = None;
let mut cur = String::new();
@@ -109,23 +107,23 @@ impl Lexer for StringLexer {
tail: &str,
err: &mut Vec<OrcErr>,
ctx: &'a LexContext<'a>,
) -> GenTokTree<'a> {
) -> GenTokTree {
let str_val_res = parse_string(&str.split_off(0));
if let Err(e) = &str_val_res {
err.push(e.clone().into_proj(ctx.pos(tail) - str.len() as u32, ctx.i).await);
err.push(e.clone().into_proj(&ctx.src, ctx.pos(tail) - str.len() as u32, ctx.i()).await);
}
let str_val = str_val_res.unwrap_or_default();
GenTok::X(IntStrAtom::from(ctx.i.i(&*str_val).await).factory())
.at(ctx.tok_ran(str.len() as u32, tail)) as GenTokTree<'a>
x_tok(IntStrAtom::from(ctx.i().i(&*str_val).await)).at(ctx.pos_lt(str.len() as u32, tail))
as GenTokTree
}
let add_frag = |prev: Option<GenTokTree<'a>>, new: GenTokTree<'a>| async {
let add_frag = |prev: Option<GenTokTree>, new: GenTokTree| async {
let Some(prev) = prev else { return new };
let concat_fn = GenTok::Reference(sym!(std::string::concat; ctx.i).await)
.at(prev.range.start..prev.range.start);
let concat_fn = ref_tok(sym!(std::string::concat; ctx.i()).await)
.at(SrcRange::zw(prev.sr.path(), prev.sr.start()));
wrap_tokv([concat_fn, prev, new])
};
loop {
if let Some(rest) = tail.strip_prefix(delim) {
if let Some(rest) = tail.strip_prefix('"') {
return Ok((rest, add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await));
} else if let Some(rest) = tail.strip_prefix('$') {
ret = Some(add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await);
@@ -143,9 +141,9 @@ impl Lexer for StringLexer {
} else {
let range = ctx.pos(all)..ctx.pos("");
return Err(mk_errv(
ctx.i.i("No string end").await,
ctx.i().i("No string end").await,
"String never terminated with \"",
[Pos::Range(range.clone()).into()],
[SrcRange::new(range.clone(), &ctx.src)],
));
}
}

15
orchid-std/src/std/string/str_lib.rs Normal file
View File

@@ -0,0 +1,15 @@
use std::rc::Rc;
use orchid_extension::tree::{GenMember, comments, fun, prefix};
use super::str_atom::StrAtom;
use crate::OrcString;
pub fn gen_str_lib() -> Vec<GenMember> {
prefix("std::string", [comments(
["Concatenate two strings"],
fun(true, "concat", |left: OrcString, right: OrcString| async move {
StrAtom::new(Rc::new(left.get_string().await.to_string() + &right.get_string().await))
}),
)])
}

2
orchid.code-workspace
View File

@@ -32,6 +32,8 @@
"rust-analyzer.rustfmt.extraArgs": [
"+nightly"
],
"rust-analyzer.cargo.features": "all",
"rust-analyzer.check.features": "all",
"files.associations": {
"*.mjsd": "markdown"
},

2
orchidlang/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "orchidlang"
version = "0.3.0"
edition = "2021"
edition = "2024"
license = "GPL-3.0"
repository = "https://github.com/lbfalvy/orchid"
description = """

2
orcx/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "orcx"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

Some files were not shown because too many files have changed in this diff Show More