Compiles again after command subsystem

terrified to start testing

orchid-extension/src/lexer.rs

@@ -7,27 +7,30 @@ use futures::FutureExt;
 use futures::future::LocalBoxFuture;
 use orchid_base::{IStr, OrcErrv, OrcRes, Pos, SrcRange, Sym, is, mk_errv};
 
-use crate::api;
-use crate::entrypoint::request;
-use crate::expr::BorrowedExprStore;
-use crate::parser::PTokTree;
 use crate::tree::GenTokTree;
+use crate::{BorrowedExprStore, PTokTree, api, request};
 
-pub async fn ekey_cascade() -> IStr { is("An error cascading from a recursive call").await }
-pub async fn ekey_not_applicable() -> IStr {
+pub(crate) async fn ekey_cascade() -> IStr { is("An error cascading from a recursive call").await }
+pub(crate) async fn ekey_not_applicable() -> IStr {
 	is("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
 }
 const MSG_INTERNAL_ERROR: &str = "This error is a sentinel for the extension library.\
 	it should not be emitted by the extension.";
 
-pub async fn err_cascade() -> OrcErrv {
+pub(crate) async fn err_cascade() -> OrcErrv {
 	mk_errv(ekey_cascade().await, MSG_INTERNAL_ERROR, [Pos::None])
 }
 
+/// Return this error if your lexer can determine that it is not applicable to
+/// this piece of syntax. This error will not be raised if another lexer
+/// matches, or if the piece of matched syntax is found to be valid until
+/// runtime
 pub async fn err_not_applicable() -> OrcErrv {
 	mk_errv(ekey_not_applicable().await, MSG_INTERNAL_ERROR, [Pos::None])
 }
 
+/// Object passed to lexers for recursion and position-related convenience
+/// methods
 pub struct LexContext<'a> {
 	pub(crate) exprs: &'a BorrowedExprStore,
 	pub text: &'a IStr,
@@ -36,7 +39,7 @@ pub struct LexContext<'a> {
 	pub(crate) src: Sym,
 }
 impl<'a> LexContext<'a> {
-	pub fn new(
+	pub(crate) fn new(
 		exprs: &'a BorrowedExprStore,
 		text: &'a IStr,
 		id: api::ParsId,
@@ -45,7 +48,11 @@ impl<'a> LexContext<'a> {
 	) -> Self {
 		Self { exprs, id, pos, src, text }
 	}
+	/// The logical path of the source file, also the path of the file's root
+	/// module
 	pub fn src(&self) -> &Sym { &self.src }
+	/// Lex an interpolated expression of some kind
+	///
 	/// This function returns [PTokTree] because it can never return
 	/// [orchid_base::Token::NewExpr]. You can use
 	/// [crate::parser::p_tree2gen] to convert this to [crate::tree::GenTokTree]
@@ -58,17 +65,23 @@ impl<'a> LexContext<'a> {
 		let tree = PTokTree::from_api(lx.tree, &mut { self.exprs }, &mut (), &self.src).await;
 		Ok((&self.text[lx.pos as usize..], tree))
 	}
-
+	/// Find the index of a cursor given the remaining, not-yet-consumed text
 	pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
+	/// Convenience method to find the source position of a token given the text
+	/// it was found in and the text after it was parsed.
 	pub fn pos_tt(&self, tail_with: &'a str, tail_without: &'a str) -> SrcRange {
 		SrcRange::new(self.pos(tail_with)..self.pos(tail_without), &self.src)
 	}
-
+	/// Convenience method to find the source position of a token given its length
+	/// and the remaining text afterwards. The length can be any number type but
+	/// must convert to a u32 without errors
 	pub fn pos_lt(&self, len: impl TryInto<u32, Error: fmt::Debug>, tail: &'a str) -> SrcRange {
 		SrcRange::new(self.pos(tail) - len.try_into().unwrap()..self.pos(tail), &self.src)
 	}
 }
 
+/// One or more tokens returned by the parser. In practice, [GenTokTree],
+/// `Vec<GenTokTree>`, or `[GenTokTree; usize]`
 pub trait LexedData {
 	fn into_vec(self) -> Vec<GenTokTree>;
 }
@@ -82,16 +95,26 @@ impl<const N: usize> LexedData for [GenTokTree; N] {
 	fn into_vec(self) -> Vec<GenTokTree> { self.to_vec() }
 }
 
+/// A lexer plugin to extend the syntax of Orchid
 pub trait Lexer: Debug + Send + Sync + Sized + Default + 'static {
+	/// As an optimization, your lexer will only receive snippets that start with
+	/// a character included in one of these ranges. If you have a multi-character
+	/// discriminator, include all possible starting chars in this and return
+	/// [err_not_applicable] if the entire discriminator was not found
 	const CHAR_FILTER: &'static [RangeInclusive<char>];
+	/// Attempt to lex some custom syntax from the start of the tail string.
+	/// Return the remaining text and the lexed tokens.
 	fn lex<'a>(
 		tail: &'a str,
 		lctx: &'a LexContext<'a>,
 	) -> impl Future<Output = OrcRes<(&'a str, impl LexedData)>>;
 }
 
+/// Type-erased [Lexer]
 pub trait DynLexer: Debug + Send + Sync + 'static {
+	/// Type-erased [Lexer::CHAR_FILTER]
 	fn char_filter(&self) -> &'static [RangeInclusive<char>];
+	/// Type-erased [Lexer::lex]
 	fn lex<'a>(
 		&self,
 		tail: &'a str,
@@ -110,4 +133,6 @@ impl<T: Lexer> DynLexer for T {
 	}
 }
 
+/// Type-erased instance of a lexer that is returned by
+/// [crate::System::lexers]
 pub type LexerObj = &'static dyn DynLexer;