Custom lexers can now terminate operators

New constraint: custom lexer output is dropped whenever it is used to terminate an operator nested inside another custom lexer, because the recursive call has to return exactly one lexeme

orchid-extension/src/entrypoint.rs

@@ -277,7 +277,11 @@ impl ExtensionBuilder {
 												return handle.reply(&lex, &eopt).await;
 											},
 											Ok((s, expr)) => {
-												let expr = expr.into_api(&mut (), &mut ()).await;
+												let expr = join_all(
+													(expr.into_iter())
+														.map(|tok| async { tok.into_api(&mut (), &mut ()).await }),
+												)
+												.await;
 												let pos = (text.len() - s.len()) as u32;
 												expr_store.dispose().await;
 												return handle.reply(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;

orchid-extension/src/lexer.rs

@@ -72,12 +72,22 @@ impl<'a> LexContext<'a> {
 	}
 }
 
+pub trait LexedData {
+	fn into_vec(self) -> Vec<GenTokTree>;
+}
+impl LexedData for GenTokTree {
+	fn into_vec(self) -> Vec<GenTokTree> { vec![self] }
+}
+impl LexedData for Vec<GenTokTree> {
+	fn into_vec(self) -> Vec<GenTokTree> { self }
+}
+
 pub trait Lexer: Debug + Send + Sync + Sized + Default + 'static {
 	const CHAR_FILTER: &'static [RangeInclusive<char>];
 	fn lex<'a>(
 		tail: &'a str,
 		lctx: &'a LexContext<'a>,
-	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
+	) -> impl Future<Output = OrcRes<(&'a str, impl LexedData)>>;
 }
 
 pub trait DynLexer: Debug + Send + Sync + 'static {
@@ -86,7 +96,7 @@ pub trait DynLexer: Debug + Send + Sync + 'static {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>>;
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -95,8 +105,8 @@ impl<T: Lexer> DynLexer for T {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>> {
-		T::lex(tail, ctx).boxed_local()
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>> {
+		async { T::lex(tail, ctx).await.map(|(s, d)| (s, d.into_vec())) }.boxed_local()
 	}
 }