retoor/orchid
1
0
Fork 0
forked from Orchid/orchid
Code Pull Requests 1 Activity

Lexer test mode works

Browse Source
This commit is contained in:
Lawrence Bethlenfalvy
2024-08-04 23:24:32 +02:00
parent 9d35ba8040
commit 11951ede43
36 changed files with 687 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
orchid-host/src/child.rs
View File

@@ -1,5 +1,5 @@
use std::sync::Mutex;
use std::{io, mem, process};
use std::{fmt, io, mem, process};
use orchid_base::msg::{recv_msg, send_msg};
@@ -7,20 +7,33 @@ pub struct SharedChild {
child: process::Child,
stdin: Mutex<process::ChildStdin>,
stdout: Mutex<process::ChildStdout>,
debug: Option<(String, Mutex<Box<dyn fmt::Write>>)>,
}
impl SharedChild {
pub fn new(cmd: &mut process::Command) -> io::Result<Self> {
let mut child = cmd.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
pub fn new(command: &mut process::Command, debug: Option<(&str, impl fmt::Write + 'static)>) -> io::Result<Self> {
let mut child = command.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
let stdin = Mutex::new(child.stdin.take().expect("Piped stdin above"));
let stdout = Mutex::new(child.stdout.take().expect("Piped stdout above"));
Ok(Self { stdin, stdout, child })
let debug = debug.map(|(n, w)| (n.to_string(), Mutex::new(Box::new(w) as Box<dyn fmt::Write>)));
Ok(Self { child, stdin, stdout, debug })
}
pub fn send_msg(&self, msg: &[u8]) -> io::Result<()> {
if let Some((n, dbg)) = &self.debug {
let mut dbg = dbg.lock().unwrap();
writeln!(dbg, "To {n}: {msg:?}").unwrap();
}
send_msg(&mut *self.stdin.lock().unwrap(), msg)
}
pub fn recv_msg(&self) -> io::Result<Vec<u8>> { recv_msg(&mut *self.stdout.lock().unwrap()) }
pub fn recv_msg(&self) -> io::Result<Vec<u8>> {
let msg = recv_msg(&mut *self.stdout.lock().unwrap());
if let Some((n, dbg)) = &self.debug {
let mut dbg = dbg.lock().unwrap();
writeln!(dbg, "From {n}: {msg:?}").unwrap();
}
msg
}
}
impl Drop for SharedChild {
fn drop(&mut self) { mem::drop(self.child.kill()) }

4
orchid-host/src/expr.rs
View File

@@ -29,7 +29,9 @@ impl RtExpr {
self.id()
}
pub fn resolve(tk: ExprTicket) -> Option<Self> { KNOWN_EXPRS.read().unwrap().get(&tk).cloned() }
pub fn from_api(api: Expr, sys: &System) -> Self { todo!() }
pub fn from_api(api: Expr, sys: &System) -> Self {
Self { data: Arc::default(), is_canonical: Arc::default() }
}
}
impl Drop for RtExpr {
fn drop(&mut self) {

125
orchid-host/src/extension.rs
View File

@@ -1,6 +1,13 @@
use std::io::Write as _;
use orchid_api::logging::Log;
use orchid_base::logging::Logger;
use orchid_base::msg::{recv_msg, send_msg};
use substack::{Stackframe, Substack};
use std::collections::VecDeque;
use std::io::{stderr, BufRead, BufReader, Write as _};
use std::num::NonZero;
use std::ops::Deref;
use std::path::PathBuf;
use std::process::ChildStdin;
use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
use std::sync::mpsc::{sync_channel, SyncSender};
use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
@@ -11,7 +18,7 @@ use hashbrown::hash_map::Entry;
use hashbrown::HashMap;
use itertools::Itertools;
use lazy_static::lazy_static;
use orchid_api::atom::{Atom, AtomDrop, AtomSame, CallRef, FinalCall, Fwd, Fwded};
use orchid_api::atom::{Atom, AtomDrop, AtomPrint, AtomSame, CallRef, FinalCall, Fwd, Fwded};
use orchid_api::error::ProjResult;
use orchid_api::expr::{Acquire, Expr, ExprNotif, ExprTicket, Release, Relocate};
use orchid_api::interner::IntReq;
@@ -83,6 +90,7 @@ impl AtomHand {
self.0.owner.reqnot().request(Fwded(self.0.api_ref(), req))
}
pub fn api_ref(&self) -> Atom { self.0.api_ref() }
pub fn print(&self) -> String { self.0.owner.reqnot().request(AtomPrint(self.0.api_ref())) }
}
/// Data held about an Extension. This is refcounted within [Extension]. It's
@@ -92,11 +100,16 @@ impl AtomHand {
#[derive(destructure)]
pub struct ExtensionData {
child: Mutex<process::Child>,
child_stdin: Mutex<ChildStdin>,
reqnot: ReqNot<HostMsgSet>,
systems: Vec<SystemCtor>,
logger: Logger,
}
impl Drop for ExtensionData {
fn drop(&mut self) { self.reqnot.notify(HostExtNotif::Exit) }
fn drop(&mut self) {
self.reqnot.notify(HostExtNotif::Exit);
self.child.lock().unwrap().wait().expect("Extension exited with error");
}
}
fn acq_expr(sys: SysId, extk: ExprTicket) {
@@ -115,27 +128,46 @@ fn rel_expr(sys: SysId, extk: ExprTicket) {
#[derive(Clone)]
pub struct Extension(Arc<ExtensionData>);
impl Extension {
pub fn new(mut cmd: process::Command) -> io::Result<Self> {
let mut child = cmd.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
HostHeader.encode(child.stdin.as_mut().unwrap());
let eh = ExtensionHeader::decode(child.stdout.as_mut().unwrap());
Ok(Self(Arc::new_cyclic(|weak| ExtensionData {
pub fn new(mut cmd: process::Command, logger: Logger) -> io::Result<Self> {
let mut child = cmd
.stdin(process::Stdio::piped())
.stdout(process::Stdio::piped())
.stderr(process::Stdio::piped())
.spawn()?;
let mut child_stdin = child.stdin.take().unwrap();
let mut child_stdout = child.stdout.take().unwrap();
let child_stderr = child.stderr.take().unwrap();
thread::Builder::new().name("stderr forwarder".to_string()).spawn(|| {
let mut reader = BufReader::new(child_stderr);
loop {
let mut buf = String::new();
if 0 == reader.read_line(&mut buf).unwrap() {
break;
}
stderr().write_all(buf.as_bytes()).unwrap();
}
}).unwrap();
HostHeader{ log_strategy: logger.strat() }.encode(&mut child_stdin);
let eh = ExtensionHeader::decode(&mut child_stdout);
let ret = Arc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
logger,
child: Mutex::new(child),
child_stdin: Mutex::new(child_stdin),
reqnot: ReqNot::new(
clone!(weak; move |sfn, _| {
let arc: Arc<ExtensionData> = weak.upgrade().unwrap();
let mut g = arc.child.lock().unwrap();
g.stdin.as_mut().unwrap().write_all(sfn).unwrap();
eprintln!("Downsending {:?}", sfn);
send_msg(&mut *weak.upgrade().unwrap().child_stdin.lock().unwrap(), sfn).unwrap();
}),
|notif, _| match notif {
clone!(weak; move |notif, _| match notif {
ExtHostNotif::ExprNotif(ExprNotif::Acquire(Acquire(sys, extk))) => acq_expr(sys, extk),
ExtHostNotif::ExprNotif(ExprNotif::Release(Release(sys, extk))) => rel_expr(sys, extk),
ExtHostNotif::ExprNotif(ExprNotif::Relocate(Relocate { dec, inc, expr })) => {
acq_expr(inc, expr);
rel_expr(dec, expr);
},
ExtHostNotif::AdviseSweep(_advice) => eprintln!("Sweep advice is unsupported")
},
ExtHostNotif::AdviseSweep(_advice) => eprintln!("Sweep advice is unsupported"),
ExtHostNotif::Log(Log(str)) => weak.upgrade().unwrap().logger.log(str),
}),
|req| match req.req() {
ExtHostReq::Ping(ping) => req.handle(ping, &()),
ExtHostReq::IntReq(IntReq::InternStr(s)) => req.handle(s, &intern(&**s.0).marker()),
@@ -160,7 +192,19 @@ impl Extension {
},
),
systems: eh.systems.into_iter().map(|decl| SystemCtor { decl, ext: weak.clone() }).collect(),
})))
});
let weak = Arc::downgrade(&ret);
let prog_pbuf = PathBuf::from(cmd.get_program());
let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy();
thread::Builder::new().name(format!("host-end:{}", prog)).spawn(move || {
loop {
let ingress = recv_msg(&mut child_stdout).expect("could not receive");
if let Some(sys) = weak.upgrade() {
sys.reqnot.receive(ingress);
}
}
}).unwrap();
Ok(Self(ret))
}
pub fn systems(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
}
@@ -293,3 +337,54 @@ impl Deref for System {
type Target = SystemInstData;
fn deref(&self) -> &Self::Target { self.0.as_ref() }
}
#[derive(Debug, Clone)]
pub enum SysResolvErr {
Loop(Vec<String>),
Missing(String)
}
pub fn init_systems(tgts: &[String], exts: &[Extension]) -> Result<Vec<System>, SysResolvErr> {
let mut to_load = HashMap::<&str, &SystemCtor>::new();
let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque::<&str>>();
while let Some(target) = to_find.pop_front() {
if to_load.contains_key(target) {
continue;
}
let ctor = (exts.iter())
.flat_map(|e| e.systems().filter(|c| c.decl.name == target))
.max_by_key(|c| c.decl.priority)
.ok_or_else(|| SysResolvErr::Missing(target.to_string()))?;
to_load.insert(target, ctor);
to_find.extend(ctor.decl.depends.iter().map(|s| s.as_str()));
}
let mut to_load_ordered = Vec::new();
fn walk_deps<'a>(
graph: &mut HashMap::<&str, &'a SystemCtor>,
list: &mut Vec<&'a SystemCtor>,
chain: Stackframe<&str>
) -> Result<(), SysResolvErr> {
if let Some(ctor) = graph.remove(chain.item) {
// if the above is none, the system is already queued. Missing systems are detected above
for dep in ctor.decl.depends.iter() {
if Substack::Frame(chain).iter().any(|c| c == dep) {
let mut circle = vec![dep.to_string()];
circle.extend(Substack::Frame(chain).iter().map(|s| s.to_string()));
return Err(SysResolvErr::Loop(circle))
}
walk_deps(graph, list, Substack::Frame(chain).new_frame(dep))?
}
list.push(ctor);
}
Ok(())
}
for tgt in tgts {
walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
}
let mut systems = HashMap::<&str, System>::new();
for ctor in to_load_ordered.iter() {
let sys = ctor.run(ctor.depends().map(|n| &systems[n]));
systems.insert(ctor.name(), sys);
}
Ok(systems.into_values().collect_vec())
}

9
orchid-host/src/lex.rs
View File

@@ -3,12 +3,12 @@ use std::num::NonZeroU64;
use hashbrown::HashMap;
use orchid_api::parser::SubLexed;
use orchid_api::system::SysId;
use orchid_api::tree::{Paren, Token, TokenTree, TreeTicket};
use orchid_api::tree::{Token, TokenTree, TreeTicket};
use orchid_base::error::OwnedError;
use orchid_base::intern;
use orchid_base::interner::{deintern, intern, Tok};
use orchid_base::location::Pos;
use orchid_base::tokens::OwnedPh;
use orchid_base::tokens::{OwnedPh, PARENS};
use crate::extension::{AtomHand, System};
use crate::results::{mk_err, OwnedResult};
@@ -69,9 +69,6 @@ impl<'a> LexCtx<'a> {
}
}
const PARENS: &[(char, char, Paren)] =
&[('(', ')', Paren::Round), ('[', ']', Paren::Square), ('{', '}', Paren::Curly)];
pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
let start = ctx.get_pos();
assert!(
@@ -164,7 +161,7 @@ pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}:\\".contains(c) }
fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}\\".contains(c) }
fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
fn tt_to_owned(api: &TokenTree, sys: SysId, ctx: &mut LexCtx<'_>) -> OwnedTokTree {

53
orchid-host/src/tree.rs
View File

@@ -1,4 +1,6 @@
use std::borrow::Borrow;
use std::cell::RefCell;
use std::fmt::{Display, Write};
use std::ops::Range;
use std::sync::{Mutex, OnceLock};
@@ -9,7 +11,7 @@ use orchid_base::error::OwnedError;
use orchid_base::interner::{deintern, Tok};
use orchid_base::location::Pos;
use orchid_base::name::Sym;
use orchid_base::tokens::OwnedPh;
use orchid_base::tokens::{OwnedPh, PARENS};
use ordered_float::NotNan;
use crate::expr::RtExpr;
@@ -49,6 +51,9 @@ impl OwnedTokTree {
tokv.into_iter().map(|t| Self::from_api(t.borrow(), sys, do_slot)).collect()
}
}
impl Display for OwnedTokTree {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
}
#[derive(Clone, Debug)]
pub enum OwnedTok {
@@ -62,6 +67,52 @@ pub enum OwnedTok {
Ph(OwnedPh),
Bottom(Vec<OwnedError>),
}
impl Display for OwnedTok {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
thread_local! {
static PAREN_LEVEL: RefCell<usize> = 0.into();
}
fn get_indent() -> usize { PAREN_LEVEL.with_borrow(|t| *t) }
fn with_indent<T>(f: impl FnOnce() -> T) -> T {
PAREN_LEVEL.with_borrow_mut(|t| *t += 1);
let r = f();
PAREN_LEVEL.with_borrow_mut(|t| *t -= 1);
r
}
match self {
Self::Atom(ah) => f.write_str(&indent(&ah.print(), get_indent(), false)),
Self::BR => write!(f, "\n{}", " ".repeat(get_indent())),
Self::Bottom(err) => write!(f, "Botttom({})",
err.iter().map(|e| format!("{}: {}", e.description, e.message)).join(", ")
),
Self::Comment(c) => write!(f, "--[{c}]--"),
Self::Lambda(arg) => with_indent(|| write!(f, "\\ {} .", fmt_tt_v(arg))),
Self::NS => f.write_str("::"),
Self::Name(n) => f.write_str(n),
Self::Ph(ph) => write!(f, "{ph}"),
Self::S(p, b) => {
let (lp, rp, _) = PARENS.iter().find(|(_, _, par)| par == p).unwrap();
f.write_char(*lp)?;
with_indent(|| f.write_str(&fmt_tt_v(b)))?;
f.write_char(*rp)
}
}
}
}
pub fn fmt_tt_v<'a>(ttv: impl IntoIterator<Item = &'a OwnedTokTree>) -> String {
ttv.into_iter().join(" ")
}
pub fn indent(s: &str, lvl: usize, first: bool) -> String {
if first {
s.replace("\n", &("\n".to_string() + &" ".repeat(lvl)))
} else if let Some((fst, rest)) = s.split_once('\n') {
fst.to_string() + "\n" + &indent(rest, lvl, true)
} else {
s.to_string()
}
}
#[derive(Debug)]
pub struct OwnedItem {