lex_hello worked for a second just now

this is just a backup however

orcx/Cargo.toml

@@ -6,8 +6,12 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-stream = "0.3.6"
 camino = "1.1.9"
 clap = { version = "4.5.24", features = ["derive"] }
+futures = "0.3.31"
 itertools = "0.14.0"
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 orchid-host = { version = "0.1.0", path = "../orchid-host" }
+substack = "1.1.1"
+tokio = { version = "1.43.0", features = ["full"] }

orcx/src/main.rs

@@ -1,17 +1,26 @@
 use std::fs::File;
 use std::io::Read;
+use std::mem;
 use std::process::Command;
-use std::sync::Arc;
+use std::rc::Rc;
 
+use async_stream::try_stream;
 use camino::Utf8PathBuf;
 use clap::{Parser, Subcommand};
-use itertools::Itertools;
-use orchid_base::interner::intern;
+use futures::{Stream, TryStreamExt, io};
+use orchid_base::clone;
+use orchid_base::error::ReporterImpl;
 use orchid_base::logging::{LogStrategy, Logger};
+use orchid_base::parse::Snippet;
 use orchid_base::tree::ttv_fmt;
-use orchid_host::extension::{Extension, init_systems};
+use orchid_host::ctx::Ctx;
+use orchid_host::extension::Extension;
 use orchid_host::lex::lex;
-use orchid_host::subprocess::Subprocess;
+use orchid_host::parse::{self, ParseCtx, ParseCtxImpl, parse_items};
+use orchid_host::subprocess::ext_command;
+use orchid_host::system::init_systems;
+use substack::Substack;
+use tokio::task::{LocalSet, spawn_local};
 
 #[derive(Parser, Debug)]
 #[command(version, about, long_about)]
@@ -30,23 +39,65 @@ pub enum Commands {
 		#[arg(short, long)]
 		file: Utf8PathBuf,
 	},
+	Parse {
+		#[arg(short, long)]
+		file: Utf8PathBuf,
+	},
 }
 
-fn main() {
-	let args = Args::parse();
-	let logger = Logger::new(LogStrategy::StdErr);
-	match args.command {
-		Commands::Lex { file } => {
-			let extensions = (args.extension.iter())
-				.map(|f| Subprocess::new(Command::new(f.as_os_str()), logger.clone()).unwrap())
-				.map(|cmd| Extension::new(Arc::new(cmd), logger.clone()).unwrap())
-				.collect_vec();
-			let systems = init_systems(&args.system, &extensions).unwrap();
-			let mut file = File::open(file.as_std_path()).unwrap();
-			let mut buf = String::new();
-			file.read_to_string(&mut buf).unwrap();
-			let lexemes = lex(intern(&buf), &systems).unwrap();
-			println!("{}", ttv_fmt(&lexemes))
-		},
+fn get_all_extensions<'a>(
+	args: &'a Args,
+	logger: &'a Logger,
+	ctx: &'a Ctx,
+) -> impl Stream<Item = io::Result<Extension>> + 'a {
+	try_stream! {
+		for exe in args.extension.iter() {
+			let init = ext_command(Command::new(exe.as_os_str()), logger.clone(), ctx.clone()).await
+				.unwrap();
+			let ext = Extension::new(init, logger.clone(), ctx.clone())?;
+			spawn_local(clone!(ext; async move { loop { ext.recv_one().await }}));
+			yield ext
+		}
 	}
 }
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() {
+	LocalSet::new()
+		.run_until(async {
+			let args = Args::parse();
+			let ctx = &Ctx::new(Rc::new(|fut| mem::drop(spawn_local(fut))));
+			let logger = Logger::new(LogStrategy::Discard);
+			let extensions =
+				get_all_extensions(&args, &logger, ctx).try_collect::<Vec<Extension>>().await.unwrap();
+			match args.command {
+				Commands::Lex { file } => {
+					let systems = init_systems(&args.system, &extensions).await.unwrap();
+					let mut file = File::open(file.as_std_path()).unwrap();
+					let mut buf = String::new();
+					file.read_to_string(&mut buf).unwrap();
+					let lexemes = lex(ctx.i.i(&buf).await, &systems, ctx).await.unwrap();
+					println!("{}", ttv_fmt(&lexemes).await)
+				},
+				Commands::Parse { file } => {
+					let systems = init_systems(&args.system, &extensions).await.unwrap();
+					let mut file = File::open(file.as_std_path()).unwrap();
+					let mut buf = String::new();
+					file.read_to_string(&mut buf).unwrap();
+					let lexemes = lex(ctx.i.i(&buf).await, &systems, ctx).await.unwrap();
+					let Some(first) = lexemes.first() else {
+						println!("File empty!");
+						return;
+					};
+					let reporter = ReporterImpl::new();
+					let pctx = ParseCtxImpl { reporter: &reporter, systems: &systems };
+					let snip = Snippet::new(first, &lexemes, &ctx.i);
+					let ptree = parse_items(&pctx, Substack::Bottom, snip).await.unwrap();
+					for item in ptree {
+						println!("{item:?}")
+					}
+				},
+			}
+		})
+		.await
+}