retoor/orchid
1
0
Fork 0
forked from Orchid/orchid
Code Pull Requests 1 Activity

Commit pending merge

Browse Source
This commit is contained in:
Lawrence Bethlenfalvy
2025-04-30 22:13:09 +02:00
parent c9b349bccf
commit 1a25f52471
18 changed files with 198 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
orchid-api/src/lexer.rs
View File

@@ -3,7 +3,7 @@ use std::ops::RangeInclusive;
use orchid_api_derive::{Coding, Hierarchy}; use orchid_api_derive::{Coding, Hierarchy};
use orchid_api_traits::Request; use orchid_api_traits::Request;
use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TokenTree}; use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TStrv, TokenTree};
/// - All ranges contain at least one character /// - All ranges contain at least one character
/// - All ranges are in increasing characeter order /// - All ranges are in increasing characeter order
@@ -18,6 +18,8 @@ pub struct LexExpr {
pub id: ParsId, pub id: ParsId,
pub text: TStr, pub text: TStr,
pub pos: u32, pub pos: u32,
/// Source root module path
pub src: TStrv,
} }
impl Request for LexExpr { impl Request for LexExpr {
type Response = Option<OrcResult<LexedExpr>>; type Response = Option<OrcResult<LexedExpr>>;

2
orchid-api/src/location.rs
View File

@@ -17,8 +17,6 @@ pub enum Location {
Gen(CodeGenInfo), Gen(CodeGenInfo),
/// Range and file /// Range and file
SourceRange(SourceRange), SourceRange(SourceRange),
/// Range only, file implied. Most notably used by parsers
Range(Range<u32>),
} }
#[derive(Clone, Debug, Coding)] #[derive(Clone, Debug, Coding)]

4
orchid-api/src/parser.rs
View File

@@ -13,7 +13,11 @@ pub struct ParsId(pub NonZeroU64);
#[extends(HostExtReq)] #[extends(HostExtReq)]
pub struct ParseLine { pub struct ParseLine {
pub sys: SysId, pub sys: SysId,
/// The immediately enclosing module path
pub module: TStrv, pub module: TStrv,
/// The root module path for the snipppet of source code, prefix of
/// [ParseLine#module]
pub src: TStrv,
pub comments: Vec<Comment>, pub comments: Vec<Comment>,
pub exported: bool, pub exported: bool,
pub line: Vec<TokenTree>, pub line: Vec<TokenTree>,

22
orchid-base/src/location.rs
View File

@@ -23,15 +23,13 @@ pub enum Pos {
Inherit, Inherit,
Gen(CodeGenInfo), Gen(CodeGenInfo),
/// Range and file /// Range and file
SourceRange(SourceRange), SrcRange(SrcRange),
/// Range only, file implied. Most notably used by parsers
Range(Range<u32>),
} }
impl Pos { impl Pos {
pub fn pretty_print(&self, get_src: &mut impl GetSrc) -> String { pub fn pretty_print(&self, get_src: &mut impl GetSrc) -> String {
match self { match self {
Self::Gen(g) => g.to_string(), Self::Gen(g) => g.to_string(),
Self::SourceRange(sr) => sr.pretty_print(&get_src(&sr.path)), Self::SrcRange(sr) => sr.pretty_print(&get_src(&sr.path)),
// Can't pretty print partial and meta-location // Can't pretty print partial and meta-location
other => format!("{other:?}"), other => format!("{other:?}"),
} }
@@ -39,17 +37,17 @@ impl Pos {
pub async fn from_api(api: &api::Location, i: &Interner) -> Self { pub async fn from_api(api: &api::Location, i: &Interner) -> Self {
match_mapping!(api, api::Location => Pos { match_mapping!(api, api::Location => Pos {
None, Inherit, SlotTarget, None, Inherit, SlotTarget,
Range(r.clone()),
Gen(cgi => CodeGenInfo::from_api(cgi, i).await), Gen(cgi => CodeGenInfo::from_api(cgi, i).await),
SourceRange(sr => SourceRange::from_api(sr, i).await) } {
api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr, i).await)
}) })
} }
pub fn to_api(&self) -> api::Location { pub fn to_api(&self) -> api::Location {
match_mapping!(self, Pos => api::Location { match_mapping!(self, Pos => api::Location {
None, Inherit, SlotTarget, None, Inherit, SlotTarget,
Range(r.clone()),
Gen(cgi.to_api()), Gen(cgi.to_api()),
SourceRange(sr.to_api()), } {
Self::SrcRange(sr) => api::Location::SourceRange(sr.to_api()),
}) })
} }
} }
@@ -57,12 +55,12 @@ impl Pos {
/// Exact source code location. Includes where the code was loaded from, what /// Exact source code location. Includes where the code was loaded from, what
/// the original source code was, and a byte range. /// the original source code was, and a byte range.
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct SourceRange { pub struct SrcRange {
pub(crate) path: Sym, pub(crate) path: Sym,
pub(crate) range: Range<u32>, pub(crate) range: Range<u32>,
} }
impl SourceRange { impl SrcRange {
pub fn new(range: &Range<u32>, path: &Sym) -> Self { pub fn new(range: Range<u32>, path: &Sym) -> Self {
Self { range: range.clone(), path: path.clone() } Self { range: range.clone(), path: path.clone() }
} }
/// Create a dud [SourceRange] for testing. Its value is unspecified and /// Create a dud [SourceRange] for testing. Its value is unspecified and
@@ -77,7 +75,7 @@ impl SourceRange {
/// 0-based index of last byte + 1 /// 0-based index of last byte + 1
pub fn end(&self) -> u32 { self.range.end } pub fn end(&self) -> u32 { self.range.end }
/// Syntactic location /// Syntactic location
pub fn pos(&self) -> Pos { Pos::SourceRange(self.clone()) } pub fn pos(&self) -> Pos { Pos::SrcRange(self.clone()) }
/// Transform the numeric byte range /// Transform the numeric byte range
pub fn map_range(&self, map: impl FnOnce(Range<u32>) -> Range<u32>) -> Self { pub fn map_range(&self, map: impl FnOnce(Range<u32>) -> Range<u32>) -> Self {
Self { range: map(self.range()), path: self.path() } Self { range: map(self.range()), path: self.path() }

12
orchid-base/src/number.rs
View File

@@ -5,7 +5,8 @@ use ordered_float::NotNan;
use crate::error::{OrcErr, mk_err}; use crate::error::{OrcErr, mk_err};
use crate::interner::Interner; use crate::interner::Interner;
use crate::location::Pos; use crate::location::SrcRange;
use crate::name::Sym;
/// A number, either floating point or unsigned int, parsed by Orchid. /// A number, either floating point or unsigned int, parsed by Orchid.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -54,7 +55,12 @@ pub struct NumError {
pub kind: NumErrorKind, pub kind: NumErrorKind,
} }
pub async fn num_to_err(NumError { kind, range }: NumError, offset: u32, i: &Interner) -> OrcErr { pub async fn num_to_err(
NumError { kind, range }: NumError,
offset: u32,
source: &Sym,
i: &Interner,
) -> OrcErr {
mk_err( mk_err(
i.i("Failed to parse number").await, i.i("Failed to parse number").await,
match kind { match kind {
@@ -62,7 +68,7 @@ pub async fn num_to_err(NumError { kind, range }: NumError, offset: u32, i: &Int
NumErrorKind::InvalidDigit => "non-digit character encountered", NumErrorKind::InvalidDigit => "non-digit character encountered",
NumErrorKind::Overflow => "The number being described is too large or too accurate", NumErrorKind::Overflow => "The number being described is too large or too accurate",
}, },
[Pos::Range(offset + range.start as u32..offset + range.end as u32).into()], [SrcRange::new(offset + range.start as u32..offset + range.end as u32, source).pos().into()],
) )
} }

55
orchid-base/src/parse.rs
View File

@@ -1,6 +1,6 @@
use std::fmt::{self, Display}; use std::fmt::{self, Display};
use std::iter; use std::iter;
use std::ops::{Deref, Range}; use std::ops::Deref;
use futures::FutureExt; use futures::FutureExt;
use futures::future::join_all; use futures::future::join_all;
@@ -10,9 +10,9 @@ use crate::api;
use crate::error::{OrcRes, Reporter, mk_err, mk_errv}; use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
use crate::format::fmt; use crate::format::fmt;
use crate::interner::{Interner, Tok}; use crate::interner::{Interner, Tok};
use crate::location::Pos; use crate::location::{Pos, SrcRange};
use crate::name::VPath; use crate::name::{VName, VPath};
use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token}; use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_range};
pub trait ParseCtx { pub trait ParseCtx {
fn i(&self) -> &Interner; fn i(&self) -> &Interner;
@@ -58,10 +58,7 @@ where
pub fn get(self, idx: u32) -> Option<&'a TokTree<A, X>> { self.cur.get(idx as usize) } pub fn get(self, idx: u32) -> Option<&'a TokTree<A, X>> { self.cur.get(idx as usize) }
pub fn len(self) -> u32 { self.cur.len() as u32 } pub fn len(self) -> u32 { self.cur.len() as u32 }
pub fn prev(self) -> &'a TokTree<A, X> { self.prev } pub fn prev(self) -> &'a TokTree<A, X> { self.prev }
pub fn pos(self) -> Range<u32> { pub fn sr(self) -> SrcRange { ttv_range(self.cur).unwrap_or_else(|| self.prev.sr.clone()) }
(self.cur.first().map(|f| f.range.start..self.cur.last().unwrap().range.end))
.unwrap_or(self.prev.range.clone())
}
pub fn pop_front(self) -> Option<(&'a TokTree<A, X>, Self)> { pub fn pop_front(self) -> Option<(&'a TokTree<A, X>, Self)> {
self.cur.first().map(|r| (r, self.split_at(1).1)) self.cur.first().map(|r| (r, self.split_at(1).1))
} }
@@ -107,7 +104,7 @@ pub fn strip_fluff<A: ExprRepr, X: ExtraTok>(tt: &TokTree<A, X>) -> Option<TokTr
Token::S(p, b) => Token::S(*p, b.iter().filter_map(strip_fluff).collect()), Token::S(p, b) => Token::S(*p, b.iter().filter_map(strip_fluff).collect()),
t => t.clone(), t => t.clone(),
}; };
Some(TokTree { tok, range: tt.range.clone() }) Some(TokTree { tok, sr: tt.sr.clone() })
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
@@ -116,6 +113,7 @@ pub struct Comment {
pub range: Range<u32>, pub range: Range<u32>,
} }
impl Comment { impl Comment {
// XXX: which of these four are actually used?
pub async fn from_api(c: &api::Comment, i: &Interner) -> Self { pub async fn from_api(c: &api::Comment, i: &Interner) -> Self {
Self { text: i.ex(c.text).await, range: c.range.clone() } Self { text: i.ex(c.text).await, range: c.range.clone() }
} }
@@ -170,10 +168,11 @@ pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
) -> ParseRes<'a, &'a TokTree<A, X>, A, X> { ) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
match snip.skip_fluff().pop_front() { match snip.skip_fluff().pop_front() {
Some((output, tail)) => Ok(Parsed { output, tail }), Some((output, tail)) => Ok(Parsed { output, tail }),
None => Err(mk_errv(ctx.i().i("Unexpected end").await, "Pattern ends abruptly", [Pos::Range( None => Err(mk_errv(
snip.pos(), ctx.i().i("Unexpected end").await,
) "Line ends abruptly; more tokens were expected",
.into()])), [snip.sr().pos().into()],
)),
} }
} }
@@ -185,7 +184,7 @@ pub async fn expect_end(
Some(surplus) => Err(mk_errv( Some(surplus) => Err(mk_errv(
ctx.i().i("Extra code after end of line").await, ctx.i().i("Extra code after end of line").await,
"Code found after the end of the line", "Code found after the end of the line",
[Pos::Range(surplus.range.clone()).into()], [surplus.sr.pos().into()],
)), )),
None => Ok(()), None => Ok(()),
} }
@@ -202,7 +201,7 @@ pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
t => Err(mk_errv( t => Err(mk_errv(
ctx.i().i("Expected specific keyword").await, ctx.i().i("Expected specific keyword").await,
format!("Expected {tok} but found {:?}", fmt(t, ctx.i()).await), format!("Expected {tok} but found {:?}", fmt(t, ctx.i()).await),
[Pos::Range(head.range.clone()).into()], [head.sr.pos().into()],
)), )),
} }
} }
@@ -217,12 +216,12 @@ pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>( pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
ctx: &impl ParseCtx, ctx: &impl ParseCtx,
tail: Snippet<'a, A, X>, tail: Snippet<'a, A, X>,
) -> ParseRes<'a, Vec<(Import, Pos)>, A, X> { ) -> ParseRes<'a, Vec<Import>, A, X> {
let Some((tt, tail)) = tail.skip_fluff().pop_front() else { let Some((tt, tail)) = tail.skip_fluff().pop_front() else {
return Err(mk_errv( return Err(mk_errv(
ctx.i().i("Expected token").await, ctx.i().i("Expected token").await,
"Expected a name, a parenthesized list of names, or a globstar.", "Expected a name, a parenthesized list of names, or a globstar.",
[Pos::Range(tail.pos()).into()], [tail.sr().pos().into()],
)); ));
}; };
let ret = rec(tt, ctx).await; let ret = rec(tt, ctx).await;
@@ -230,8 +229,8 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
pub async fn rec<A: ExprRepr, X: ExtraTok>( pub async fn rec<A: ExprRepr, X: ExtraTok>(
tt: &TokTree<A, X>, tt: &TokTree<A, X>,
ctx: &impl ParseCtx, ctx: &impl ParseCtx,
) -> OrcRes<Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>> { ) -> OrcRes<Vec<(Vec<Tok<String>>, Option<Tok<String>>, SrcRange)>> {
let ttpos = Pos::Range(tt.range.clone()); let ttpos = tt.sr.pos();
match &tt.tok { match &tt.tok {
Token::NS(ns, body) => { Token::NS(ns, body) => {
if !ns.starts_with(name_start) { if !ns.starts_with(name_start) {
@@ -247,7 +246,7 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
Token::Name(ntok) => { Token::Name(ntok) => {
let n = ntok; let n = ntok;
let nopt = Some(n.clone()); let nopt = Some(n.clone());
Ok(vec![(vec![], nopt, Pos::Range(tt.range.clone()))]) Ok(vec![(vec![], nopt, tt.sr.clone())])
}, },
Token::S(Paren::Round, b) => { Token::S(Paren::Round, b) => {
let mut o = Vec::new(); let mut o = Vec::new();
@@ -272,19 +271,29 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
} }
ret.map(|output| { ret.map(|output| {
let output = (output.into_iter()) let output = (output.into_iter())
.map(|(p, name, pos)| (Import { path: VPath::new(p.into_iter().rev()), name }, pos)) .map(|(p, name, sr)| Import { path: VPath::new(p.into_iter().rev()), name, sr })
.collect_vec(); .collect_vec();
Parsed { output, tail } Parsed { output, tail }
}) })
} }
/// A compound name, possibly ending with a globstar /// A compound name, possibly ending with a globstar. It cannot be just a
/// globstar; either the name has to be known or the path has to be non-empty.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Import { pub struct Import {
pub path: VPath, pub path: VPath,
pub name: Option<Tok<String>>, pub name: Option<Tok<String>>,
pub sr: SrcRange,
}
impl Import {
/// Most specific concrete path
pub fn mspath(self) -> VName {
match self.name {
Some(n) => self.path.name_with_suffix(n),
None => self.path.into_name().expect("Import cannot be empty"),
}
}
} }
impl Display for Import { impl Display for Import {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str())) write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str()))

53
orchid-base/src/tree.rs
View File

@@ -2,7 +2,6 @@ use std::borrow::Borrow;
use std::fmt::{self, Debug, Display}; use std::fmt::{self, Debug, Display};
use std::future::Future; use std::future::Future;
use std::marker::PhantomData; use std::marker::PhantomData;
use std::ops::Range;
use std::rc::Rc; use std::rc::Rc;
use async_stream::stream; use async_stream::stream;
@@ -16,7 +15,8 @@ use trait_set::trait_set;
use crate::error::OrcErrv; use crate::error::OrcErrv;
use crate::format::{FmtCtx, FmtUnit, Format, Variants}; use crate::format::{FmtCtx, FmtUnit, Format, Variants};
use crate::interner::{Interner, Tok}; use crate::interner::{Interner, Tok};
use crate::location::Pos; use crate::location::{Pos, SrcRange};
use crate::name::Sym;
use crate::parse::Snippet; use crate::parse::Snippet;
use crate::{api, match_mapping, tl_cache}; use crate::{api, match_mapping, tl_cache};
@@ -26,7 +26,7 @@ pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::
fn from_api( fn from_api(
api: &ApiEquiv, api: &ApiEquiv,
ctx: &mut Self::FromApiCtx<'_>, ctx: &mut Self::FromApiCtx<'_>,
pos: Pos, pos: SrcRange,
i: &Interner, i: &Interner,
) -> impl Future<Output = Self>; ) -> impl Future<Output = Self>;
fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>; fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>;
@@ -34,7 +34,7 @@ pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::
impl<T: Clone + Debug + Coding> TokenVariant<T> for Never { impl<T: Clone + Debug + Coding> TokenVariant<T> for Never {
type FromApiCtx<'a> = (); type FromApiCtx<'a> = ();
type ToApiCtx<'a> = (); type ToApiCtx<'a> = ();
async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: Pos, _: &Interner) -> Self { async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange, _: &Interner) -> Self {
panic!("Cannot deserialize Never") panic!("Cannot deserialize Never")
} }
async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} } async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} }
@@ -55,7 +55,7 @@ pub fn recur<H: ExprRepr, X: ExtraTok>(
tt: TokTree<H, X>, tt: TokTree<H, X>,
f: &impl Fn(TokTree<H, X>, &dyn RecurCB<H, X>) -> TokTree<H, X>, f: &impl Fn(TokTree<H, X>, &dyn RecurCB<H, X>) -> TokTree<H, X>,
) -> TokTree<H, X> { ) -> TokTree<H, X> {
f(tt, &|TokTree { range, tok }| { f(tt, &|TokTree { sr: range, tok }| {
let tok = match tok { let tok = match tok {
tok @ (Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::Name(_)) => tok, tok @ (Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::Name(_)) => tok,
tok @ (Token::Handle(_) | Token::NewExpr(_)) => tok, tok @ (Token::Handle(_) | Token::NewExpr(_)) => tok,
@@ -64,7 +64,7 @@ pub fn recur<H: ExprRepr, X: ExtraTok>(
Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()), Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()), Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
}; };
TokTree { range, tok } TokTree { sr: range, tok }
}) })
} }
@@ -94,28 +94,33 @@ impl Display for TokHandle<'_> {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct TokTree<H: ExprRepr, X: ExtraTok> { pub struct TokTree<H: ExprRepr, X: ExtraTok> {
pub tok: Token<H, X>, pub tok: Token<H, X>,
pub range: Range<u32>, /// The protocol has a Range<u32> because these are always transmitted in the
/// context of a given snippet, but internal logic and error reporting is
/// easier if the in-memory representation also includes the snippet path.
pub sr: SrcRange,
} }
impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> { impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
pub async fn from_api( pub async fn from_api(
tt: &api::TokenTree, tt: &api::TokenTree,
hctx: &mut H::FromApiCtx<'_>, hctx: &mut H::FromApiCtx<'_>,
xctx: &mut X::FromApiCtx<'_>, xctx: &mut X::FromApiCtx<'_>,
src: &Sym,
i: &Interner, i: &Interner,
) -> Self { ) -> Self {
let pos = SrcRange::new(tt.range.clone(), src);
let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> { let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> {
BR, BR,
NS(n => Tok::from_api(*n, i).await, NS(n => Tok::from_api(*n, i).await,
b => Box::new(Self::from_api(b, hctx, xctx, i).boxed_local().await)), b => Box::new(Self::from_api(b, hctx, xctx, src, i).boxed_local().await)),
Bottom(e => OrcErrv::from_api(e, i).await), Bottom(e => OrcErrv::from_api(e, i).await),
LambdaHead(arg => ttv_from_api(arg, hctx, xctx, i).await), LambdaHead(arg => ttv_from_api(arg, hctx, xctx, src, i).await),
Name(n => Tok::from_api(*n, i).await), Name(n => Tok::from_api(*n, i).await),
S(*par, b => ttv_from_api(b, hctx, xctx, i).await), S(*par, b => ttv_from_api(b, hctx, xctx, src, i).await),
Comment(c.clone()), Comment(c.clone()),
NewExpr(expr => X::from_api(expr, xctx, Pos::Range(tt.range.clone()), i).await), NewExpr(expr => X::from_api(expr, xctx, pos.clone(), i).await),
Handle(tk => H::from_api(tk, hctx, Pos::Range(tt.range.clone()), i).await) Handle(tk => H::from_api(tk, hctx, pos.clone(), i).await)
}); });
Self { range: tt.range.clone(), tok } Self { sr: pos, tok }
} }
pub async fn into_api( pub async fn into_api(
@@ -134,7 +139,7 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
Handle(hand.into_api(hctx).await), Handle(hand.into_api(hctx).await),
NewExpr(expr.into_api(xctx).await), NewExpr(expr.into_api(xctx).await),
}); });
api::TokenTree { range: self.range.clone(), token } api::TokenTree { range: self.sr.range.clone(), token }
} }
pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) } pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
@@ -152,8 +157,9 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
} }
pub fn is_fluff(&self) -> bool { matches!(self.tok, Token::Comment(_) | Token::BR) } pub fn is_fluff(&self) -> bool { matches!(self.tok, Token::Comment(_) | Token::BR) }
pub fn lambda(arg: Vec<Self>, mut body: Vec<Self>) -> Self { pub fn lambda(arg: Vec<Self>, mut body: Vec<Self>) -> Self {
let arg_range = ttv_range(&arg); let arg_range = ttv_range(&arg).expect("Lambda with empty arg!");
let s_range = arg_range.start..body.last().expect("Lambda with empty body!").range.end; let mut s_range = arg_range.clone();
s_range.range.end = body.last().expect("Lambda with empty body!").sr.range.end;
body.insert(0, Token::LambdaHead(arg).at(arg_range)); body.insert(0, Token::LambdaHead(arg).at(arg_range));
Token::S(Paren::Round, body).at(s_range) Token::S(Paren::Round, body).at(s_range)
} }
@@ -168,11 +174,12 @@ pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>, tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
hctx: &mut H::FromApiCtx<'_>, hctx: &mut H::FromApiCtx<'_>,
xctx: &mut X::FromApiCtx<'_>, xctx: &mut X::FromApiCtx<'_>,
src: &Sym,
i: &Interner, i: &Interner,
) -> Vec<TokTree<H, X>> { ) -> Vec<TokTree<H, X>> {
stream! { stream! {
for tok in tokv { for tok in tokv {
yield TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, i).boxed_local().await yield TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await
} }
} }
.collect() .collect()
@@ -201,8 +208,8 @@ pub fn wrap_tokv<H: ExprRepr, X: ExtraTok>(
0 => panic!("A tokv with no elements is illegal"), 0 => panic!("A tokv with no elements is illegal"),
1 => items_v.into_iter().next().unwrap(), 1 => items_v.into_iter().next().unwrap(),
_ => { _ => {
let range = items_v.first().unwrap().range.start..items_v.last().unwrap().range.end; let sr = ttv_range(&items_v).expect("empty handled above");
Token::S(api::Paren::Round, items_v).at(range) Token::S(api::Paren::Round, items_v).at(sr)
}, },
} }
} }
@@ -237,7 +244,7 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
Bottom(OrcErrv), Bottom(OrcErrv),
} }
impl<H: ExprRepr, X: ExtraTok> Token<H, X> { impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
pub fn at(self, range: Range<u32>) -> TokTree<H, X> { TokTree { range, tok: self } } pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) } pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) }
pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> { pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
match self { match self {
@@ -273,9 +280,9 @@ impl<H: ExprRepr, X: ExtraTok> Format for Token<H, X> {
} }
} }
pub fn ttv_range<'a>(ttv: &[TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>]) -> Range<u32> { pub fn ttv_range<'a>(ttv: &[TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>]) -> Option<SrcRange> {
assert!(!ttv.is_empty(), "Empty slice has no range"); let range = ttv.first()?.sr.range.start..ttv.last().unwrap().sr.range.end;
ttv.first().unwrap().range.start..ttv.last().unwrap().range.end Some(SrcRange { path: ttv.first().unwrap().sr.path(), range })
} }
pub async fn ttv_fmt<'a: 'b, 'b>( pub async fn ttv_fmt<'a: 'b, 'b>(

10
orchid-extension/src/entrypoint.rs
View File

@@ -268,10 +268,11 @@ pub fn extension_init(
let vfs = systems_g[sys_id].vfses[vfs_id].load(&path, ctx).await; let vfs = systems_g[sys_id].vfses[vfs_id].load(&path, ctx).await;
hand.handle(&vfs_read, &vfs).await hand.handle(&vfs_read, &vfs).await
}, },
api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, text, pos, id }) => { api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) => {
let sys_ctx = get_ctx(sys).await; let sys_ctx = get_ctx(sys).await;
let text = Tok::from_api(text, &i).await; let text = Tok::from_api(text, &i).await;
let ctx = LexContext { id, pos, text: &text, ctx: sys_ctx.clone() }; let src = Sym::from_api(src, sys_ctx.i()).await;
let ctx = LexContext { id, pos, text: &text, src, ctx: sys_ctx.clone() };
let trigger_char = text.chars().nth(pos as usize).unwrap(); let trigger_char = text.chars().nth(pos as usize).unwrap();
let err_na = err_not_applicable(&i).await; let err_na = err_not_applicable(&i).await;
let err_cascade = err_cascade(&i).await; let err_cascade = err_cascade(&i).await;
@@ -294,11 +295,12 @@ pub fn extension_init(
hand.handle(&lex, &None).await hand.handle(&lex, &None).await
}, },
api::HostExtReq::ParseLine(pline) => { api::HostExtReq::ParseLine(pline) => {
let api::ParseLine { module, exported, comments, sys, line } = &pline; let api::ParseLine { module, src, exported, comments, sys, line } = &pline;
let mut ctx = get_ctx(*sys).await; let mut ctx = get_ctx(*sys).await;
let parsers = ctx.cted().inst().dyn_parsers(); let parsers = ctx.cted().inst().dyn_parsers();
let src = Sym::from_api(*src, ctx.i()).await;
let comments = join_all(comments.iter().map(|c| Comment::from_api(c, &i))).await; let comments = join_all(comments.iter().map(|c| Comment::from_api(c, &i))).await;
let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx, &mut (), &i).await; let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx, &mut (), &src, &i).await;
let snip = Snippet::new(line.first().expect("Empty line"), &line); let snip = Snippet::new(line.first().expect("Empty line"), &line);
let (head, tail) = snip.pop_front().unwrap(); let (head, tail) = snip.pop_front().unwrap();
let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") }; let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") };

13
orchid-extension/src/lexer.rs
View File

@@ -1,11 +1,12 @@
use std::future::Future; use std::future::Future;
use std::ops::{Range, RangeInclusive}; use std::ops::RangeInclusive;
use futures::FutureExt; use futures::FutureExt;
use futures::future::LocalBoxFuture; use futures::future::LocalBoxFuture;
use orchid_base::error::{OrcErr, OrcRes, mk_err}; use orchid_base::error::{OrcErr, OrcRes, mk_err};
use orchid_base::interner::{Interner, Tok}; use orchid_base::interner::{Interner, Tok};
use orchid_base::location::Pos; use orchid_base::location::{Pos, SrcRange};
use orchid_base::name::Sym;
use orchid_base::reqnot::Requester; use orchid_base::reqnot::Requester;
use crate::api; use crate::api;
@@ -34,6 +35,7 @@ pub struct LexContext<'a> {
pub text: &'a Tok<String>, pub text: &'a Tok<String>,
pub id: api::ParsId, pub id: api::ParsId,
pub pos: u32, pub pos: u32,
pub src: Sym,
} }
impl<'a> LexContext<'a> { impl<'a> LexContext<'a> {
pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree)> { pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree)> {
@@ -41,14 +43,15 @@ impl<'a> LexContext<'a> {
let Some(lx) = self.ctx.reqnot().request(api::SubLex { pos: start, id: self.id }).await else { let Some(lx) = self.ctx.reqnot().request(api::SubLex { pos: start, id: self.id }).await else {
return Err(err_cascade(self.ctx.i()).await.into()); return Err(err_cascade(self.ctx.i()).await.into());
}; };
let tree = GenTokTree::from_api(&lx.tree, &mut self.ctx.clone(), &mut (), self.ctx.i()).await; let tree =
GenTokTree::from_api(&lx.tree, &mut self.ctx.clone(), &mut (), &self.src, self.ctx.i()).await;
Ok((&self.text[lx.pos as usize..], tree)) Ok((&self.text[lx.pos as usize..], tree))
} }
pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 } pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
pub fn tok_ran(&self, len: u32, tail: &'a str) -> Range<u32> { pub fn tok_ran(&self, len: u32, tail: &'a str) -> SrcRange {
self.pos(tail) - len..self.pos(tail) SrcRange::new(self.pos(tail) - len..self.pos(tail), &self.src)
} }
} }

6
orchid-extension/src/tree.rs
View File

@@ -8,7 +8,7 @@ use futures::{FutureExt, StreamExt};
use hashbrown::HashMap; use hashbrown::HashMap;
use itertools::Itertools; use itertools::Itertools;
use orchid_base::interner::{Interner, Tok}; use orchid_base::interner::{Interner, Tok};
use orchid_base::location::Pos; use orchid_base::location::SrcRange;
use orchid_base::name::Sym; use orchid_base::name::Sym;
use orchid_base::reqnot::ReqHandlish; use orchid_base::reqnot::ReqHandlish;
use orchid_base::tree::{TokTree, Token, TokenVariant}; use orchid_base::tree::{TokTree, Token, TokenVariant};
@@ -32,7 +32,7 @@ impl TokenVariant<api::Expression> for GExpr {
async fn from_api( async fn from_api(
_: &api::Expression, _: &api::Expression,
_: &mut Self::FromApiCtx<'_>, _: &mut Self::FromApiCtx<'_>,
_: Pos, _: SrcRange,
_: &Interner, _: &Interner,
) -> Self { ) -> Self {
panic!("Received new expression from host") panic!("Received new expression from host")
@@ -47,7 +47,7 @@ impl TokenVariant<api::ExprTicket> for Expr {
async fn from_api( async fn from_api(
api: &api::ExprTicket, api: &api::ExprTicket,
ctx: &mut Self::FromApiCtx<'_>, ctx: &mut Self::FromApiCtx<'_>,
_: Pos, _: SrcRange,
_: &Interner, _: &Interner,
) -> Self { ) -> Self {
// SAFETY: receiving trees from sublexers implies ownership transfer // SAFETY: receiving trees from sublexers implies ownership transfer

14
orchid-host/src/dealias.rs
View File

@@ -1,10 +1,7 @@
use std::rc::Rc;
use futures::FutureExt; use futures::FutureExt;
use hashbrown::{HashMap, HashSet}; use hashbrown::{HashMap, HashSet};
use itertools::{Either, Itertools}; use itertools::{Either, Itertools};
use orchid_base::error::{OrcErr, Reporter, mk_err}; use orchid_base::error::{OrcErr, Reporter, mk_err};
use orchid_base::format::{FmtCtxImpl, Format, take_first};
use orchid_base::interner::{Interner, Tok}; use orchid_base::interner::{Interner, Tok};
use orchid_base::location::Pos; use orchid_base::location::Pos;
use orchid_base::name::{NameLike, Sym, VName}; use orchid_base::name::{NameLike, Sym, VName};
@@ -125,18 +122,17 @@ pub async fn imports_to_aliases(
match &item.kind { match &item.kind {
ItemKind::Import(imp) => match absolute_path(cwd, &imp.path) { ItemKind::Import(imp) => match absolute_path(cwd, &imp.path) {
Err(e) => Err(e) =>
ctx.rep.report(e.err_obj(ctx.i, item.pos.clone(), &imp.path.iter().join("::")).await), ctx.rep.report(e.err_obj(ctx.i, item.sr.pos(), &imp.path.iter().join("::")).await),
Ok(abs_path) => { Ok(abs_path) => {
let names = match imp.name.as_ref() { let names = match imp.name.as_ref() {
Some(n) => Either::Right([n.clone()].into_iter()), Some(n) => Either::Right([n.clone()].into_iter()),
None => Either::Left( None =>
resolv_glob(cwd, root, &abs_path, item.pos.clone(), ctx).await.into_iter(), Either::Left(resolv_glob(cwd, root, &abs_path, item.sr.pos(), ctx).await.into_iter()),
),
}; };
for name in names { for name in names {
let mut tgt = abs_path.clone().suffix([name.clone()]).to_sym(ctx.i).await; let mut tgt = abs_path.clone().suffix([name.clone()]).to_sym(ctx.i).await;
let src = Sym::new(cwd.iter().cloned().chain([name]), ctx.i).await.unwrap(); let src = Sym::new(cwd.iter().cloned().chain([name]), ctx.i).await.unwrap();
import_locs.entry(src.clone()).or_insert(vec![]).push(item.pos.clone()); import_locs.entry(src.clone()).or_insert(vec![]).push(item.sr.pos());
if let Some(tgt2) = alias_map.get(&tgt) { if let Some(tgt2) = alias_map.get(&tgt) {
tgt = tgt2.clone(); tgt = tgt2.clone();
} }
@@ -144,7 +140,7 @@ pub async fn imports_to_aliases(
ctx.rep.report(mk_err( ctx.rep.report(mk_err(
ctx.i.i("Circular references").await, ctx.i.i("Circular references").await,
format!("{src} circularly refers to itself"), format!("{src} circularly refers to itself"),
[item.pos.clone().into()], [item.sr.pos().into()],
)); ));
continue; continue;
} }

7
orchid-host/src/extension.rs
View File

@@ -20,6 +20,7 @@ use orchid_base::clone;
use orchid_base::format::{FmtCtxImpl, Format}; use orchid_base::format::{FmtCtxImpl, Format};
use orchid_base::interner::Tok; use orchid_base::interner::Tok;
use orchid_base::logging::Logger; use orchid_base::logging::Logger;
use orchid_base::name::Sym;
use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _}; use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
use crate::api; use crate::api;
@@ -203,6 +204,7 @@ impl Extension {
pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>( pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
&self, &self,
source: Tok<String>, source: Tok<String>,
src: Sym,
pos: u32, pos: u32,
sys: api::SysId, sys: api::SysId,
mut r: impl FnMut(u32) -> F, mut r: impl FnMut(u32) -> F,
@@ -214,8 +216,9 @@ impl Extension {
self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
let (ret, ()) = join( let (ret, ()) = join(
async { async {
let res = let res = (self.reqnot())
(self.reqnot()).request(api::LexExpr { id, pos, sys, text: source.to_api() }).await; .request(api::LexExpr { id, pos, sys, src: src.to_api(), text: source.to_api() })
.await;
// collect sender to unblock recursion handler branch before returning // collect sender to unblock recursion handler branch before returning
self.0.lex_recur.lock().await.remove(&id); self.0.lex_recur.lock().await.remove(&id);
res res

30
orchid-host/src/lex.rs
View File

@@ -4,7 +4,8 @@ use async_std::sync::Mutex;
use futures::FutureExt; use futures::FutureExt;
use orchid_base::error::{OrcErrv, OrcRes, mk_errv}; use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
use orchid_base::interner::Tok; use orchid_base::interner::Tok;
use orchid_base::location::Pos; use orchid_base::location::SrcRange;
use orchid_base::name::Sym;
use orchid_base::parse::{name_char, name_start, op_char, unrep_space}; use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
use orchid_base::tokens::PARENS; use orchid_base::tokens::PARENS;
use orchid_base::tree::recur; use orchid_base::tree::recur;
@@ -18,6 +19,7 @@ use crate::system::System;
pub struct LexCtx<'a> { pub struct LexCtx<'a> {
pub systems: &'a [System], pub systems: &'a [System],
pub source: &'a Tok<String>, pub source: &'a Tok<String>,
pub path: &'a Sym,
pub tail: &'a str, pub tail: &'a str,
pub sub_trees: &'a mut Vec<Expr>, pub sub_trees: &'a mut Vec<Expr>,
pub ctx: &'a Ctx, pub ctx: &'a Ctx,
@@ -27,6 +29,7 @@ impl<'a> LexCtx<'a> {
where 'a: 'b { where 'a: 'b {
LexCtx { LexCtx {
source: self.source, source: self.source,
path: self.path,
tail: &self.source[pos as usize..], tail: &self.source[pos as usize..],
systems: self.systems, systems: self.systems,
sub_trees: &mut *self.sub_trees, sub_trees: &mut *self.sub_trees,
@@ -49,7 +52,7 @@ impl<'a> LexCtx<'a> {
let mut exprs = self.ctx.common_exprs.clone(); let mut exprs = self.ctx.common_exprs.clone();
let foo = recur(subtree, &|tt, r| { let foo = recur(subtree, &|tt, r| {
if let ParsTok::NewExpr(expr) = tt.tok { if let ParsTok::NewExpr(expr) = tt.tok {
return ParsTok::Handle(expr).at(tt.range); return ParsTok::Handle(expr).at(tt.sr);
} }
r(tt) r(tt)
}); });
@@ -60,6 +63,7 @@ impl<'a> LexCtx<'a> {
&tree, &tree,
&mut self.ctx.common_exprs.clone(), &mut self.ctx.common_exprs.clone(),
&mut ExprParseCtx { ctx: self.ctx.clone(), exprs: self.ctx.common_exprs.clone() }, &mut ExprParseCtx { ctx: self.ctx.clone(), exprs: self.ctx.common_exprs.clone() },
self.path,
&self.ctx.i, &self.ctx.i,
) )
.await .await
@@ -103,7 +107,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
return Err(mk_errv( return Err(mk_errv(
ctx.ctx.i.i("Unterminated block comment").await, ctx.ctx.i.i("Unterminated block comment").await,
"This block comment has no ending ]--", "This block comment has no ending ]--",
[Pos::Range(start..start + 3).into()], [SrcRange::new(start..start + 3, ctx.path).pos().into()],
)); ));
}; };
ctx.set_tail(tail); ctx.set_tail(tail);
@@ -120,7 +124,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
return Err(mk_errv( return Err(mk_errv(
ctx.ctx.i.i("Unclosed lambda").await, ctx.ctx.i.i("Unclosed lambda").await,
"Lambdae started with \\ should separate arguments from body with .", "Lambdae started with \\ should separate arguments from body with .",
[Pos::Range(start..start + 1).into()], [SrcRange::new(start..start + 1, ctx.path).pos().into()],
)); ));
} }
arg.push(lex_once(ctx).boxed_local().await?); arg.push(lex_once(ctx).boxed_local().await?);
@@ -135,7 +139,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
return Err(mk_errv( return Err(mk_errv(
ctx.ctx.i.i("unclosed paren").await, ctx.ctx.i.i("unclosed paren").await,
format!("this {lp} has no matching {rp}"), format!("this {lp} has no matching {rp}"),
[Pos::Range(start..start + 1).into()], [SrcRange::new(start..start + 1, ctx.path).pos().into()],
)); ));
} }
body.push(lex_once(ctx).boxed_local().await?); body.push(lex_once(ctx).boxed_local().await?);
@@ -153,7 +157,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
.lex(source, pos, |pos| async move { .lex(source, pos, |pos| async move {
let mut ctx_g = ctx_lck.lock().await; let mut ctx_g = ctx_lck.lock().await;
match lex_once(&mut ctx_g.push(pos)).boxed_local().await { match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
Ok(t) => Some(api::SubLexed { pos: t.range.end, tree: ctx_g.ser_subtree(t).await }), Ok(t) => Some(api::SubLexed { pos: t.sr.end(), tree: ctx_g.ser_subtree(t).await }),
Err(e) => { Err(e) => {
errors_lck.lock().await.push(e); errors_lck.lock().await.push(e);
None None
@@ -185,16 +189,22 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
return Err(mk_errv( return Err(mk_errv(
ctx.ctx.i.i("Unrecognized character").await, ctx.ctx.i.i("Unrecognized character").await,
"The following syntax is meaningless.", "The following syntax is meaningless.",
[Pos::Range(start..start + 1).into()], [SrcRange::new(start..start + 1, ctx.path).pos().into()],
)); ));
} }
}; };
Ok(ParsTokTree { tok, range: start..ctx.get_pos() }) Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
} }
pub async fn lex(text: Tok<String>, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> { pub async fn lex(
text: Tok<String>,
path: Sym,
systems: &[System],
ctx: &Ctx,
) -> OrcRes<Vec<ParsTokTree>> {
let mut sub_trees = Vec::new(); let mut sub_trees = Vec::new();
let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, ctx }; let mut ctx =
LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };
let mut tokv = Vec::new(); let mut tokv = Vec::new();
ctx.trim(unrep_space); ctx.trim(unrep_space);
while !ctx.tail.is_empty() { while !ctx.tail.is_empty() {

36
orchid-host/src/parse.rs
View File

@@ -7,7 +7,6 @@ use itertools::Itertools;
use orchid_base::error::{OrcRes, Reporter, mk_err, mk_errv}; use orchid_base::error::{OrcRes, Reporter, mk_err, mk_errv};
use orchid_base::format::fmt; use orchid_base::format::fmt;
use orchid_base::interner::{Interner, Tok}; use orchid_base::interner::{Interner, Tok};
use orchid_base::location::Pos;
use orchid_base::name::Sym; use orchid_base::name::Sym;
use orchid_base::parse::{ use orchid_base::parse::{
Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname, Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
@@ -25,6 +24,7 @@ type ParsSnippet<'a> = Snippet<'a, Expr, Expr>;
pub struct HostParseCtxImpl<'a> { pub struct HostParseCtxImpl<'a> {
pub ctx: Ctx, pub ctx: Ctx,
pub src: Sym,
pub systems: &'a [System], pub systems: &'a [System],
pub reporter: &'a Reporter, pub reporter: &'a Reporter,
pub interner: &'a Interner, pub interner: &'a Interner,
@@ -77,19 +77,19 @@ pub async fn parse_item(
expect_end(ctx, tail).await?; expect_end(ctx, tail).await?;
let mut ok = Vec::new(); let mut ok = Vec::new();
for tt in body { for tt in body {
let pos = Pos::Range(tt.range.clone()); let sr = tt.sr.clone();
match &tt.tok { match &tt.tok {
Token::Name(n) => Token::Name(n) =>
ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n.clone()) }), ok.push(Item { comments: comments.clone(), sr, kind: ItemKind::Export(n.clone()) }),
Token::NS(..) => ctx.reporter().report(mk_err( Token::NS(..) => ctx.reporter().report(mk_err(
ctx.i().i("Compound export").await, ctx.i().i("Compound export").await,
"Cannot export compound names (names containing the :: separator)", "Cannot export compound names (names containing the :: separator)",
[pos.into()], [sr.pos().into()],
)), )),
t => ctx.reporter().report(mk_err( t => ctx.reporter().report(mk_err(
ctx.i().i("Invalid export").await, ctx.i().i("Invalid export").await,
format!("Invalid export target {}", fmt(t, ctx.i()).await), format!("Invalid export target {}", fmt(t, ctx.i()).await),
[pos.into()], [sr.pos().into()],
)), )),
} }
} }
@@ -99,14 +99,14 @@ pub async fn parse_item(
Parsed { output, tail: _ } => Err(mk_errv( Parsed { output, tail: _ } => Err(mk_errv(
ctx.i().i("Malformed export").await, ctx.i().i("Malformed export").await,
"`export` can either prefix other lines or list names inside ( )", "`export` can either prefix other lines or list names inside ( )",
[Pos::Range(output.range.clone()).into()], [output.sr.pos().into()],
)), )),
}, },
n if *n == ctx.i().i("import").await => { n if *n == ctx.i().i("import").await => {
let imports = parse_import(ctx, postdisc).await?; let imports = parse_import(ctx, postdisc).await?;
Ok(Vec::from_iter(imports.into_iter().map(|(t, pos)| Item { Ok(Vec::from_iter(imports.into_iter().map(|t| Item {
comments: comments.clone(), comments: comments.clone(),
pos, sr: t.sr.clone(),
kind: ItemKind::Import(t), kind: ItemKind::Import(t),
}))) })))
}, },
@@ -115,7 +115,7 @@ pub async fn parse_item(
Some(_) => Err(mk_errv( Some(_) => Err(mk_errv(
ctx.i().i("Expected a line type").await, ctx.i().i("Expected a line type").await,
"All lines must begin with a keyword", "All lines must begin with a keyword",
[Pos::Range(item.pos()).into()], [item.sr().pos().into()],
)), )),
None => unreachable!("These lines are filtered and aggregated in earlier stages"), None => unreachable!("These lines are filtered and aggregated in earlier stages"),
} }
@@ -124,7 +124,7 @@ pub async fn parse_item(
pub async fn parse_import<'a>( pub async fn parse_import<'a>(
ctx: &impl HostParseCtx, ctx: &impl HostParseCtx,
tail: ParsSnippet<'a>, tail: ParsSnippet<'a>,
) -> OrcRes<Vec<(Import, Pos)>> { ) -> OrcRes<Vec<Import>> {
let Parsed { output: imports, tail } = parse_multiname(ctx, tail).await?; let Parsed { output: imports, tail } = parse_multiname(ctx, tail).await?;
expect_end(ctx, tail).await?; expect_end(ctx, tail).await?;
Ok(imports) Ok(imports)
@@ -153,10 +153,10 @@ pub async fn parse_exportable_item<'a>(
return Err(mk_errv( return Err(mk_errv(
ctx.i().i("Unrecognized line type").await, ctx.i().i("Unrecognized line type").await,
format!("Line types are: const, mod, macro, grammar, {ext_lines}"), format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
[Pos::Range(tail.prev().range.clone()).into()], [tail.prev().sr.pos().into()],
)); ));
}; };
Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }]) Ok(vec![Item { comments, sr: tail.sr(), kind }])
} }
pub async fn parse_module<'a>( pub async fn parse_module<'a>(
@@ -170,7 +170,7 @@ pub async fn parse_module<'a>(
return Err(mk_errv( return Err(mk_errv(
ctx.i().i("Missing module name").await, ctx.i().i("Missing module name").await,
format!("A name was expected, {} was found", fmt(output, ctx.i()).await), format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
[Pos::Range(output.range.clone()).into()], [output.sr.pos().into()],
)); ));
}, },
}; };
@@ -180,7 +180,7 @@ pub async fn parse_module<'a>(
return Err(mk_errv( return Err(mk_errv(
ctx.i().i("Expected module body").await, ctx.i().i("Expected module body").await,
format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await), format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await),
[Pos::Range(output.range.clone()).into()], [output.sr.pos().into()],
)); ));
}; };
let path = path.push(name.clone()); let path = path.push(name.clone());
@@ -197,7 +197,7 @@ pub async fn parse_const<'a>(
return Err(mk_errv( return Err(mk_errv(
ctx.i().i("Missing module name").await, ctx.i().i("Missing module name").await,
format!("A name was expected, {} was found", fmt(output, ctx.i()).await), format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
[Pos::Range(output.range.clone()).into()], [output.sr.pos().into()],
)); ));
}; };
let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?; let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?;
@@ -205,7 +205,7 @@ pub async fn parse_const<'a>(
return Err(mk_errv( return Err(mk_errv(
ctx.i().i("Missing = separator").await, ctx.i().i("Missing = separator").await,
format!("Expected = , found {}", fmt(output, ctx.i()).await), format!("Expected = , found {}", fmt(output, ctx.i()).await),
[Pos::Range(output.range.clone()).into()], [output.sr.pos().into()],
)); ));
} }
try_pop_no_fluff(ctx, tail).await?; try_pop_no_fluff(ctx, tail).await?;
@@ -223,11 +223,11 @@ pub async fn parse_expr(
.or_else(|| tail.iter().enumerate().rev().find(|(_, tt)| !tt.is_fluff())) .or_else(|| tail.iter().enumerate().rev().find(|(_, tt)| !tt.is_fluff()))
else { else {
return Err(mk_errv(ctx.i().i("Empty expression").await, "Expression ends abruptly here", [ return Err(mk_errv(ctx.i().i("Empty expression").await, "Expression ends abruptly here", [
Pos::Range(tail.pos()).into(), tail.sr().pos().into(),
])); ]));
}; };
let (function, value) = tail.split_at(last_idx as u32); let (function, value) = tail.split_at(last_idx as u32);
let pos = Pos::Range(tail.pos()); let pos = tail.sr().pos();
if !function.iter().all(TokTree::is_fluff) { if !function.iter().all(TokTree::is_fluff) {
let (f_psb, x_psb) = psb.split(); let (f_psb, x_psb) = psb.split();
let x_expr = parse_expr(ctx, path.clone(), x_psb, value).boxed_local().await?; let x_expr = parse_expr(ctx, path.clone(), x_psb, value).boxed_local().await?;

22
orchid-host/src/parsed.rs
View File

@@ -1,4 +1,3 @@
use std::cell::RefCell;
use std::fmt::Debug; use std::fmt::Debug;
use std::rc::Rc; use std::rc::Rc;
@@ -12,7 +11,7 @@ use itertools::Itertools;
use orchid_base::error::{OrcRes, mk_errv}; use orchid_base::error::{OrcRes, mk_errv};
use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants}; use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
use orchid_base::interner::Tok; use orchid_base::interner::Tok;
use orchid_base::location::Pos; use orchid_base::location::{Pos, SrcRange};
use orchid_base::name::{NameLike, Sym}; use orchid_base::name::{NameLike, Sym};
use orchid_base::parse::{Comment, Import}; use orchid_base::parse::{Comment, Import};
use orchid_base::tl_cache; use orchid_base::tl_cache;
@@ -37,7 +36,7 @@ impl TokenVariant<api::ExprTicket> for Expr {
async fn from_api( async fn from_api(
api: &api::ExprTicket, api: &api::ExprTicket,
ctx: &mut Self::FromApiCtx<'_>, ctx: &mut Self::FromApiCtx<'_>,
_: Pos, _: SrcRange,
_: &orchid_base::interner::Interner, _: &orchid_base::interner::Interner,
) -> Self { ) -> Self {
let expr = ctx.get_expr(*api).expect("Dangling expr"); let expr = ctx.get_expr(*api).expect("Dangling expr");
@@ -51,7 +50,7 @@ impl TokenVariant<api::Expression> for Expr {
async fn from_api( async fn from_api(
api: &api::Expression, api: &api::Expression,
ctx: &mut Self::FromApiCtx<'_>, ctx: &mut Self::FromApiCtx<'_>,
_: Pos, _: SrcRange,
_: &orchid_base::interner::Interner, _: &orchid_base::interner::Interner,
) -> Self { ) -> Self {
Expr::from_api(api, PathSetBuilder::new(), ctx).await Expr::from_api(api, PathSetBuilder::new(), ctx).await
@@ -76,7 +75,7 @@ impl<'a> ParsedFromApiCx<'a> {
#[derive(Debug)] #[derive(Debug)]
pub struct Item { pub struct Item {
pub pos: Pos, pub sr: SrcRange,
pub comments: Vec<Comment>, pub comments: Vec<Comment>,
pub kind: ItemKind, pub kind: ItemKind,
} }
@@ -88,8 +87,9 @@ pub enum ItemKind {
Import(Import), Import(Import),
} }
impl ItemKind { impl ItemKind {
pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } } pub fn at(self, sr: SrcRange) -> Item { Item { comments: vec![], sr, kind: self } }
} }
impl Format for Item { impl Format for Item {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit { async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
let comment_text = self.comments.iter().join("\n"); let comment_text = self.comments.iter().join("\n");
@@ -135,7 +135,6 @@ pub enum ParsedMemberKind {
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct ParsedModule { pub struct ParsedModule {
pub imports: Vec<Sym>,
pub exports: Vec<Tok<String>>, pub exports: Vec<Tok<String>>,
pub items: Vec<Item>, pub items: Vec<Item>,
} }
@@ -148,7 +147,7 @@ impl ParsedModule {
_ => None, _ => None,
}) })
.collect_vec(); .collect_vec();
Self { imports: vec![], exports, items } Self { exports, items }
} }
pub fn merge(&mut self, other: ParsedModule) { pub fn merge(&mut self, other: ParsedModule) {
let mut swap = ParsedModule::default(); let mut swap = ParsedModule::default();
@@ -178,6 +177,10 @@ impl ParsedModule {
} }
Ok(cur) Ok(cur)
} }
pub fn get_imports(&self) -> impl IntoIterator<Item = &Import> {
(self.items.iter())
.filter_map(|it| if let ItemKind::Import(i) = &it.kind { Some(i) } else { None })
}
} }
#[derive(Clone, Debug, Hash, PartialEq, Eq)] #[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub enum WalkErrorKind { pub enum WalkErrorKind {
@@ -192,8 +195,7 @@ pub struct WalkError {
} }
impl Format for ParsedModule { impl Format for ParsedModule {
async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit { async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
let import_str = self.imports.iter().map(|i| format!("import {i}")).join("\n"); let head_str = format!("export ::({})\n", self.exports.iter().join(", "));
let head_str = format!("{import_str}\nexport ::({})\n", self.exports.iter().join(", "));
Variants::sequence(self.items.len() + 1, "\n", None).units( Variants::sequence(self.items.len() + 1, "\n", None).units(
[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await), [head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
) )

29
orchid-host/src/tree.rs
View File

@@ -10,6 +10,7 @@ use orchid_base::name::Sym;
use crate::api; use crate::api;
use crate::ctx::Ctx; use crate::ctx::Ctx;
use crate::dealias::absolute_path;
use crate::expr::Expr; use crate::expr::Expr;
use crate::parsed::{ParsedMemberKind, ParsedModule}; use crate::parsed::{ParsedMemberKind, ParsedModule};
use crate::system::System; use crate::system::System;
@@ -37,6 +38,34 @@ impl Module {
} }
Self { members } Self { members }
} }
async fn walk(&self, mut path: impl Iterator<Item = Tok<String>>, ) -> &Self { todo!()}
async fn from_parsed(
parsed: &ParsedModule,
path: Sym,
parsed_root_path: Sym,
parsed_root: &ParsedModule,
root: &Module,
preload: &mut HashMap<Sym, Module>,
) -> Self {
let mut imported_names = Vec::new();
for import in parsed.get_imports() {
if let Some(n) = import.name.clone() {
imported_names.push(n);
continue;
}
// the path in a wildcard import has to be a module
if import.path.is_empty() {
panic!("Imported root")
}
if let Some(subpath) = import.path.strip_prefix(&parsed_root_path) {
let abs = absolute_path(&path, subpath);
// path is in parsed_root
} else {
// path is in root
}
}
todo!()
}
} }
pub struct Member { pub struct Member {

4
orchid-std/src/macros/rule/build.rs
View File

@@ -110,7 +110,7 @@ mod test {
use orchid_api::PhKind; use orchid_api::PhKind;
use orchid_base::interner::Interner; use orchid_base::interner::Interner;
use orchid_base::location::SourceRange; use orchid_base::location::SrcRange;
use orchid_base::sym; use orchid_base::sym;
use orchid_base::tokens::Paren; use orchid_base::tokens::Paren;
use orchid_base::tree::Ph; use orchid_base::tree::Ph;
@@ -124,7 +124,7 @@ mod test {
spin_on(async { spin_on(async {
let i = Interner::new_master(); let i = Interner::new_master();
let ex = |tok: MacTok| async { let ex = |tok: MacTok| async {
MacTree { tok: Rc::new(tok), pos: SourceRange::mock(&i).await.pos() } MacTree { tok: Rc::new(tok), pos: SrcRange::mock(&i).await.pos() }
}; };
let pattern = vec![ let pattern = vec![
ex(MacTok::Ph(Ph { ex(MacTok::Ph(Ph {

4
orchid-std/src/std/string/str_lexer.rs
View File

@@ -118,8 +118,8 @@ impl Lexer for StringLexer {
} }
let add_frag = |prev: Option<GenTokTree<'a>>, new: GenTokTree<'a>| async { let add_frag = |prev: Option<GenTokTree<'a>>, new: GenTokTree<'a>| async {
let Some(prev) = prev else { return new }; let Some(prev) = prev else { return new };
let concat_fn = GenTok::Reference(sym!(std::string::concat; ctx.i).await) let concat_fn =
.at(prev.range.start..prev.range.start); GenTok::Reference(sym!(std::string::concat; ctx.i).await).at(prev.sr.start..prev.sr.start);
wrap_tokv([concat_fn, prev, new]) wrap_tokv([concat_fn, prev, new])
}; };
loop { loop {