retoor/orchid
1
0
Fork 0
forked from Orchid/orchid
Code Pull Requests 1 Activity

Custom lexers can now terminate operators

Browse Source 
New constraint: custom lexer output is dropped whenever it is used to terminate an operator nested inside another custom lexer, because the recursive call has to return exactly one lexeme
This commit is contained in:
Lawrence Bethlenfalvy
2026-01-25 17:52:18 +01:00
parent b9f1bb74d7
commit c461f82de1
17 changed files with 333 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
orchid-extension/src/entrypoint.rs
View File

@@ -277,7 +277,11 @@ impl ExtensionBuilder {
return handle.reply(&lex, &eopt).await;
},
Ok((s, expr)) => {
let expr = expr.into_api(&mut (), &mut ()).await;
let expr = join_all(
(expr.into_iter())
.map(|tok| async { tok.into_api(&mut (), &mut ()).await }),
)
.await;
let pos = (text.len() - s.len()) as u32;
expr_store.dispose().await;
return handle.reply(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;

18
orchid-extension/src/lexer.rs
View File

@@ -72,12 +72,22 @@ impl<'a> LexContext<'a> {
}
}
pub trait LexedData {
fn into_vec(self) -> Vec<GenTokTree>;
}
impl LexedData for GenTokTree {
fn into_vec(self) -> Vec<GenTokTree> { vec![self] }
}
impl LexedData for Vec<GenTokTree> {
fn into_vec(self) -> Vec<GenTokTree> { self }
}
pub trait Lexer: Debug + Send + Sync + Sized + Default + 'static {
const CHAR_FILTER: &'static [RangeInclusive<char>];
fn lex<'a>(
tail: &'a str,
lctx: &'a LexContext<'a>,
) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
) -> impl Future<Output = OrcRes<(&'a str, impl LexedData)>>;
}
pub trait DynLexer: Debug + Send + Sync + 'static {
@@ -86,7 +96,7 @@ pub trait DynLexer: Debug + Send + Sync + 'static {
&self,
tail: &'a str,
ctx: &'a LexContext<'a>,
) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>>;
) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>>;
}
impl<T: Lexer> DynLexer for T {
@@ -95,8 +105,8 @@ impl<T: Lexer> DynLexer for T {
&self,
tail: &'a str,
ctx: &'a LexContext<'a>,
) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>> {
T::lex(tail, ctx).boxed_local()
) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>> {
async { T::lex(tail, ctx).await.map(|(s, d)| (s, d.into_vec())) }.boxed_local()
}
}