Orchid/orchid
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generic mutation scheduling system

Browse Source 
IO adapted to use it
Also, Atoms can now dispatch type-erased requests
This commit is contained in:
Lawrence Bethlenfalvy
2023-09-14 22:54:42 +01:00
parent 8c866967a9
commit 3c0056c2db
51 changed files with 991 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/systems/asynch/mod.rs
View File

@@ -1,5 +1,9 @@
mod system;
mod types;
//! An event queue other systems can use to trigger events on the main
//! interpreter thread. These events are handled when the Orchid code returns
//! `system::async::yield`, and may cause additional Orchid code to be executed
//! beyond being general Rust functions.
//! It also exposes timers.
pub use system::{AsynchConfig, InfiniteBlock};
pub use types::{Asynch, MessagePort};
mod system;
pub use system::{AsynchSystem, InfiniteBlock, MessagePort};

75
src/systems/asynch/system.rs
View File

@@ -1,5 +1,6 @@
use std::any::{type_name, Any, TypeId};
use std::cell::RefCell;
use std::collections::VecDeque;
use std::fmt::{Debug, Display};
use std::rc::Rc;
use std::sync::mpsc::Sender;
@@ -8,16 +9,15 @@ use std::time::Duration;
use hashbrown::HashMap;
use ordered_float::NotNan;
use super::types::MessagePort;
use super::Asynch;
use crate::facade::{IntoSystem, System};
use crate::foreign::cps_box::{init_cps, CPSBox};
use crate::foreign::ExternError;
use crate::foreign::{Atomic, ExternError};
use crate::interpreted::ExprInst;
use crate::interpreter::HandlerTable;
use crate::systems::codegen::call;
use crate::systems::stl::Boolean;
use crate::utils::{unwrap_or, PollEvent, Poller};
use crate::utils::poller::{PollEvent, Poller};
use crate::utils::unwrap_or;
use crate::{atomic_inert, define_fn, ConstTree, Interner};
#[derive(Debug, Clone)]
@@ -45,7 +45,7 @@ impl Debug for CancelTimer {
#[derive(Clone, Debug)]
struct Yield;
atomic_inert!(Yield, "a yield command");
atomic_inert!(Yield, typestr = "a yield command");
/// Error indicating a yield command when all event producers and timers had
/// exited
@@ -59,43 +59,43 @@ impl Display for InfiniteBlock {
}
}
impl MessagePort for Sender<Box<dyn Any + Send>> {
fn send<T: Send + 'static>(&mut self, message: T) {
let _ = Self::send(self, Box::new(message));
/// A thread-safe handle that can be used to send events of any type
#[derive(Clone)]
pub struct MessagePort(Sender<Box<dyn Any + Send>>);
impl MessagePort {
/// Send an event. Any type is accepted, handlers are dispatched by type ID
pub fn send<T: Send + 'static>(&mut self, message: T) {
let _ = self.0.send(Box::new(message));
}
}
impl<F> MessagePort for F
where
F: FnMut(Box<dyn Any + Send>) + Send + Clone + 'static,
{
fn send<T: Send + 'static>(&mut self, message: T) {
self(Box::new(message))
}
}
type AnyHandler<'a> = Box<dyn FnMut(Box<dyn Any>) -> Vec<ExprInst> + 'a>;
type AnyHandler<'a> = Box<dyn FnMut(Box<dyn Any>) -> Option<ExprInst> + 'a>;
/// Datastructures the asynch system will eventually be constructed from
pub struct AsynchConfig<'a> {
/// Datastructures the asynch system will eventually be constructed from.
pub struct AsynchSystem<'a> {
poller: Poller<Box<dyn Any + Send>, ExprInst, ExprInst>,
sender: Sender<Box<dyn Any + Send>>,
handlers: HashMap<TypeId, AnyHandler<'a>>,
}
impl<'a> AsynchConfig<'a> {
impl<'a> AsynchSystem<'a> {
/// Create a new async event loop that allows registering handlers and taking
/// references to the port before it's converted into a [System]
pub fn new() -> Self {
let (sender, poller) = Poller::new();
Self { poller, sender, handlers: HashMap::new() }
}
}
impl<'a> Asynch for AsynchConfig<'a> {
type Port = Sender<Box<dyn Any + Send>>;
fn register<T: 'static>(
/// Register a callback to be called on the owning thread when an object of
/// the given type is found on the queue. Each type should signify a single
/// command so each type should have exactly one handler.
///
/// # Panics
///
/// if the given type is already handled.
pub fn register<T: 'static>(
&mut self,
mut f: impl FnMut(Box<T>) -> Option<ExprInst> + 'a,
mut f: impl FnMut(Box<T>) -> Vec<ExprInst> + 'a,
) {
let cb = move |a: Box<dyn Any>| f(a.downcast().expect("keyed by TypeId"));
let prev = self.handlers.insert(TypeId::of::<T>(), Box::new(cb));
@@ -106,18 +106,21 @@ impl<'a> Asynch for AsynchConfig<'a> {
)
}
fn get_port(&self) -> Self::Port {
self.sender.clone()
/// Obtain a message port for sending messages to the main thread. If an
/// object is passed to the MessagePort that does not have a handler, the
/// main thread panics.
pub fn get_port(&self) -> MessagePort {
MessagePort(self.sender.clone())
}
}
impl<'a> Default for AsynchConfig<'a> {
impl<'a> Default for AsynchSystem<'a> {
fn default() -> Self {
Self::new()
}
}
impl<'a> IntoSystem<'a> for AsynchConfig<'a> {
impl<'a> IntoSystem<'a> for AsynchSystem<'a> {
fn into_system(self, i: &Interner) -> System<'a> {
let Self { mut handlers, poller, .. } = self;
let mut handler_table = HandlerTable::new();
@@ -143,7 +146,11 @@ impl<'a> IntoSystem<'a> for AsynchConfig<'a> {
});
handler_table.register({
let polly = polly.clone();
let mut microtasks = VecDeque::new();
move |_: &Yield| {
if let Some(expr) = microtasks.pop_front() {
return Ok(expr);
}
let mut polly = polly.borrow_mut();
loop {
let next = unwrap_or!(polly.run();
@@ -157,8 +164,12 @@ impl<'a> IntoSystem<'a> for AsynchConfig<'a> {
.unwrap_or_else(|| {
panic!("Unhandled messgae type: {:?}", ev.type_id())
});
if let Some(expr) = handler(ev) {
return Ok(expr);
let events = handler(ev);
// we got new microtasks
if !events.is_empty() {
microtasks = VecDeque::from(events);
// trampoline
return Ok(Yield.atom_exi());
}
},
}

30
src/systems/asynch/types.rs
View File

@@ -1,30 +0,0 @@
use crate::interpreted::ExprInst;
/// A thread-safe handle that can be used to send events of any type
pub trait MessagePort: Send + Clone + 'static {
/// Send an event. Any type is accepted, handlers are dispatched by type ID
fn send<T: Send + 'static>(&mut self, message: T);
}
pub trait Asynch {
/// A thread-safe handle that can be used to push events into the dispatcher
type Port: MessagePort;
/// Register a function that will be called synchronously when an event of the
/// accepted type is dispatched. Only one handler may be specified for each
/// event type. The handler may choose to process the event autonomously, or
/// return an Orchid thunk for the interpreter to execute.
///
/// # Panics
///
/// When the function is called with an argument type it was previously called
/// with
fn register<T: 'static>(
&mut self,
f: impl FnMut(Box<T>) -> Option<ExprInst> + 'static,
);
/// Return a handle that can be passed to worker threads and used to push
/// events onto the dispatcher
fn get_port(&self) -> Self::Port;
}

12
src/systems/io/bindings.rs
View File

@@ -33,7 +33,7 @@ define_fn! {
n: u64
} => Ok(init_cps(3, IOCmdHandlePack{
cmd: ReadCmd::RBytes(BRead::N((*n).try_into().unwrap())),
handle: *stream
handle: stream.clone()
}))
}
define_fn! {
@@ -47,7 +47,7 @@ define_fn! {
))?;
Ok(init_cps(3, IOCmdHandlePack{
cmd: ReadCmd::RBytes(BRead::Until(delim)),
handle: *stream
handle: stream.clone()
}))
}
}
@@ -57,7 +57,7 @@ define_fn! {
string: OrcString
} => Ok(init_cps(3, IOCmdHandlePack {
cmd: WriteCmd::WStr(string.get_string()),
handle: *stream,
handle: stream.clone(),
}))
}
define_fn! {
@@ -66,7 +66,7 @@ define_fn! {
bytes: Binary
} => Ok(init_cps(3, IOCmdHandlePack {
cmd: WriteCmd::WBytes(bytes.clone()),
handle: *stream
handle: stream.clone(),
}))
}
define_fn! {
@@ -76,9 +76,9 @@ define_fn! {
}))
}
pub fn io_bindings(
pub fn io_bindings<'a>(
i: &Interner,
std_streams: impl IntoIterator<Item = (&'static str, Box<dyn Atomic>)>,
std_streams: impl IntoIterator<Item = (&'a str, Box<dyn Atomic>)>,
) -> ConstTree {
ConstTree::namespace(
[i.i("system"), i.i("io")],

35
src/systems/io/facade.rs
View File

@@ -17,7 +17,7 @@ use crate::foreign::{Atomic, ExternError};
use crate::interpreter::HandlerTable;
use crate::pipeline::file_loader::embed_to_map;
use crate::sourcefile::{FileEntry, FileEntryKind, Import};
use crate::systems::asynch::{Asynch, MessagePort};
use crate::systems::asynch::AsynchSystem;
use crate::{Interner, Location};
trait_set! {
@@ -32,25 +32,25 @@ struct IOEmbed;
/// A registry that stores IO streams and executes blocking operations on them
/// in a distinct thread pool
pub struct IOSystem<P: MessagePort, ST: StreamTable> {
read_system: Rc<RefCell<ReadManager<P>>>,
write_system: Rc<RefCell<WriteManager<P>>>,
pub struct IOSystem<ST: StreamTable> {
read_system: Rc<RefCell<ReadManager>>,
write_system: Rc<RefCell<WriteManager>>,
global_streams: ST,
}
impl<P: MessagePort, ST: StreamTable> IOSystem<P, ST> {
impl<ST: StreamTable> IOSystem<ST> {
fn new(
mut get_port: impl FnMut() -> P,
asynch: &AsynchSystem,
on_sink_close: Option<Box<dyn FnMut(Sink)>>,
on_source_close: Option<Box<dyn FnMut(Source)>>,
global_streams: ST,
) -> Self {
Self {
read_system: Rc::new(RefCell::new(IOManager::new(
get_port(),
asynch.get_port(),
on_source_close,
))),
write_system: Rc::new(RefCell::new(IOManager::new(
get_port(),
asynch.get_port(),
on_sink_close,
))),
global_streams,
@@ -93,26 +93,19 @@ pub enum IOStream {
/// takes a generic parameter which is initialized from an existential in the
/// [AsynchConfig].
pub fn io_system(
asynch: &'_ mut impl Asynch,
asynch: &'_ mut AsynchSystem,
on_sink_close: Option<Box<dyn FnMut(Sink)>>,
on_source_close: Option<Box<dyn FnMut(Source)>>,
std_streams: impl IntoIterator<Item = (&'static str, IOStream)>,
) -> IOSystem<impl MessagePort, impl StreamTable> {
let this = IOSystem::new(
|| asynch.get_port(),
on_sink_close,
on_source_close,
std_streams,
);
) -> IOSystem<impl StreamTable> {
let this = IOSystem::new(asynch, on_sink_close, on_source_close, std_streams);
let (r, w) = (this.read_system.clone(), this.write_system.clone());
asynch.register(move |event| r.borrow_mut().dispatch(*event));
asynch.register(move |event| w.borrow_mut().dispatch(*event));
asynch.register(move |event| vec![r.borrow_mut().dispatch(*event)]);
asynch.register(move |event| vec![w.borrow_mut().dispatch(*event)]);
this
}
impl<'a, P: MessagePort, ST: StreamTable + 'a> IntoSystem<'a>
for IOSystem<P, ST>
{
impl<'a, ST: StreamTable + 'a> IntoSystem<'a> for IOSystem<ST> {
fn into_system(self, i: &Interner) -> System<'a> {
let (r, w) = (self.read_system.clone(), self.write_system.clone());
let mut handlers = HandlerTable::new();

154
src/systems/io/flow.rs
View File

@@ -1,22 +1,13 @@
use std::collections::VecDeque;
use std::fmt::Display;
use hashbrown::HashMap;
use crate::foreign::ExternError;
use crate::systems::asynch::MessagePort;
use crate::thread_pool::{Task, ThreadPool};
use crate::utils::take_with_output;
use crate::systems::scheduler::Canceller;
pub trait StreamHandle: Clone + Send {
fn new(id: usize) -> Self;
fn id(&self) -> usize;
}
pub trait IOHandler<Cmd: IOCmd> {
pub trait IOHandler<T> {
type Product;
fn handle(self, result: Cmd::Result) -> Self::Product;
fn handle(self, result: T) -> Self::Product;
fn early_cancel(self) -> Self::Product;
}
pub trait IOResult: Send {
@@ -26,33 +17,16 @@ pub trait IOResult: Send {
fn handle(self, handler: Self::Handler) -> Self::HandlerProduct;
}
pub struct IOEvent<Cmd: IOCmd> {
pub result: Cmd::Result,
pub stream: Cmd::Stream,
pub handle: Cmd::Handle,
}
pub trait IOCmd: Send {
type Stream: Send;
type Result: Send;
type Handle: StreamHandle;
type Handle;
fn execute(self, stream: &mut Self::Stream) -> Self::Result;
}
pub struct IOTask<P: MessagePort, Cmd: IOCmd> {
pub cmd: Cmd,
pub stream: Cmd::Stream,
pub handle: Cmd::Handle,
pub port: P,
}
impl<P: MessagePort, Cmd: IOCmd + 'static> Task for IOTask<P, Cmd> {
fn run(self) {
let Self { cmd, handle, mut port, mut stream } = self;
let result = cmd.execute(&mut stream);
port.send(IOEvent::<Cmd> { handle, result, stream })
}
fn execute(
self,
stream: &mut Self::Stream,
cancel: Canceller,
) -> Self::Result;
}
#[derive(Debug, Clone)]
@@ -61,11 +35,6 @@ pub struct IOCmdHandlePack<Cmd: IOCmd> {
pub handle: Cmd::Handle,
}
enum StreamState<Cmd: IOCmd, H: IOHandler<Cmd>> {
Free(Cmd::Stream),
Busy { handler: H, queue: VecDeque<(Cmd, H)>, closing: bool },
}
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub struct NoActiveStream(usize);
impl ExternError for NoActiveStream {}
@@ -74,106 +43,3 @@ impl Display for NoActiveStream {
write!(f, "The stream {} had already been closed", self.0)
}
}
pub struct IOManager<P: MessagePort, Cmd: IOCmd + 'static, H: IOHandler<Cmd>> {
next_id: usize,
streams: HashMap<usize, StreamState<Cmd, H>>,
on_close: Option<Box<dyn FnMut(Cmd::Stream)>>,
thread_pool: ThreadPool<IOTask<P, Cmd>>,
port: P,
}
impl<P: MessagePort, Cmd: IOCmd, H: IOHandler<Cmd>> IOManager<P, Cmd, H> {
pub fn new(port: P, on_close: Option<Box<dyn FnMut(Cmd::Stream)>>) -> Self {
Self {
next_id: 0,
streams: HashMap::new(),
thread_pool: ThreadPool::new(),
on_close,
port,
}
}
pub fn add_stream(&mut self, stream: Cmd::Stream) -> Cmd::Handle {
let id = self.next_id;
self.next_id += 1;
self.streams.insert(id, StreamState::Free(stream));
Cmd::Handle::new(id)
}
fn dispose_stream(&mut self, stream: Cmd::Stream) {
match &mut self.on_close {
Some(f) => f(stream),
None => drop(stream),
}
}
pub fn close_stream(
&mut self,
handle: Cmd::Handle,
) -> Result<(), NoActiveStream> {
let state =
(self.streams.remove(&handle.id())).ok_or(NoActiveStream(handle.id()))?;
match state {
StreamState::Free(stream) => self.dispose_stream(stream),
StreamState::Busy { handler, queue, closing } => {
let new_state = StreamState::Busy { handler, queue, closing: true };
self.streams.insert(handle.id(), new_state);
if closing {
return Err(NoActiveStream(handle.id()));
}
},
}
Ok(())
}
pub fn command(
&mut self,
handle: Cmd::Handle,
cmd: Cmd,
new_handler: H,
) -> Result<(), NoActiveStream> {
let state_mut = (self.streams.get_mut(&handle.id()))
.ok_or(NoActiveStream(handle.id()))?;
take_with_output(state_mut, |state| match state {
StreamState::Busy { closing: true, .. } =>
(state, Err(NoActiveStream(handle.id()))),
StreamState::Busy { handler, mut queue, closing: false } => {
queue.push_back((cmd, new_handler));
(StreamState::Busy { handler, queue, closing: false }, Ok(()))
},
StreamState::Free(stream) => {
let task = IOTask { cmd, stream, handle, port: self.port.clone() };
self.thread_pool.submit(task);
let new_state = StreamState::Busy {
handler: new_handler,
queue: VecDeque::new(),
closing: false,
};
(new_state, Ok(()))
},
})
}
pub fn dispatch(&mut self, event: IOEvent<Cmd>) -> Option<H::Product> {
let IOEvent { handle, result, stream } = event;
let id = handle.id();
let state =
(self.streams.remove(&id)).expect("Event dispatched on unknown stream");
let (handler, mut queue, closing) = match state {
StreamState::Busy { handler, queue, closing } =>
(handler, queue, closing),
_ => panic!("Event dispatched but the source isn't locked"),
};
if let Some((cmd, handler)) = queue.pop_front() {
let port = self.port.clone();
self.thread_pool.submit(IOTask { handle, stream, cmd, port });
self.streams.insert(id, StreamState::Busy { handler, queue, closing });
} else if closing {
self.dispose_stream(stream)
} else {
self.streams.insert(id, StreamState::Free(stream));
};
Some(handler.handle(result))
}
}

78
src/systems/io/instances.rs
View File

@@ -1,38 +1,19 @@
use std::io::{self, BufRead, BufReader, Read, Write};
use std::sync::Arc;
use super::flow::{IOCmd, IOHandler, IOManager, StreamHandle};
use super::flow::IOCmd;
use crate::foreign::Atomic;
use crate::interpreted::ExprInst;
use crate::systems::codegen::call;
use crate::systems::scheduler::{Canceller, SharedHandle};
use crate::systems::stl::Binary;
use crate::{atomic_inert, Literal};
use crate::Literal;
pub type Source = BufReader<Box<dyn Read + Send>>;
pub type Sink = Box<dyn Write + Send>;
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
pub struct SourceHandle(usize);
atomic_inert!(SourceHandle, "an input stream handle");
impl StreamHandle for SourceHandle {
fn new(id: usize) -> Self {
Self(id)
}
fn id(&self) -> usize {
self.0
}
}
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
pub struct SinkHandle(usize);
atomic_inert!(SinkHandle, "an output stream handle");
impl StreamHandle for SinkHandle {
fn new(id: usize) -> Self {
Self(id)
}
fn id(&self) -> usize {
self.0
}
}
pub type SourceHandle = SharedHandle<Source>;
pub type SinkHandle = SharedHandle<Sink>;
/// String reading command
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
@@ -62,7 +43,11 @@ impl IOCmd for ReadCmd {
// This is a buggy rule, check manually
#[allow(clippy::read_zero_byte_vec)]
fn execute(self, stream: &mut Self::Stream) -> Self::Result {
fn execute(
self,
stream: &mut Self::Stream,
_cancel: Canceller,
) -> Self::Result {
match self {
Self::RBytes(bread) => {
let mut buf = Vec::new();
@@ -93,19 +78,17 @@ pub enum ReadResult {
RStr(SRead, io::Result<String>),
RBin(BRead, io::Result<Vec<u8>>),
}
impl IOHandler<ReadCmd> for (ExprInst, ExprInst) {
type Product = ExprInst;
fn handle(self, result: ReadResult) -> Self::Product {
let (succ, fail) = self;
match result {
impl ReadResult {
pub fn dispatch(self, succ: ExprInst, fail: ExprInst) -> Vec<ExprInst> {
match self {
ReadResult::RBin(_, Err(e)) | ReadResult::RStr(_, Err(e)) =>
call(fail, vec![wrap_io_error(e)]).wrap(),
ReadResult::RBin(_, Ok(bytes)) =>
call(succ, vec![Binary(Arc::new(bytes)).atom_cls().wrap()]).wrap(),
vec![call(fail, vec![wrap_io_error(e)]).wrap()],
ReadResult::RBin(_, Ok(bytes)) => {
let arg = Binary(Arc::new(bytes)).atom_cls().wrap();
vec![call(succ, vec![arg]).wrap()]
},
ReadResult::RStr(_, Ok(text)) =>
call(succ, vec![Literal::Str(text.into()).into()]).wrap(),
vec![call(succ, vec![Literal::Str(text.into()).into()]).wrap()],
}
}
}
@@ -116,8 +99,6 @@ fn wrap_io_error(_e: io::Error) -> ExprInst {
Literal::Uint(0u64).into()
}
pub type ReadManager<P> = IOManager<P, ReadCmd, (ExprInst, ExprInst)>;
/// Writing command (string or binary)
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum WriteCmd {
@@ -131,7 +112,11 @@ impl IOCmd for WriteCmd {
type Handle = SinkHandle;
type Result = WriteResult;
fn execute(self, stream: &mut Self::Stream) -> Self::Result {
fn execute(
self,
stream: &mut Self::Stream,
_cancel: Canceller,
) -> Self::Result {
let result = match &self {
Self::Flush => stream.flush(),
Self::WStr(str) => write!(stream, "{}", str).map(|_| ()),
@@ -145,16 +130,11 @@ pub struct WriteResult {
pub cmd: WriteCmd,
pub result: io::Result<()>,
}
impl IOHandler<WriteCmd> for (ExprInst, ExprInst) {
type Product = ExprInst;
fn handle(self, result: WriteResult) -> Self::Product {
let (succ, fail) = self;
match result.result {
Ok(_) => succ,
Err(e) => call(fail, vec![wrap_io_error(e)]).wrap(),
impl WriteResult {
pub fn dispatch(self, succ: ExprInst, fail: ExprInst) -> Vec<ExprInst> {
match self.result {
Ok(_) => vec![succ],
Err(e) => vec![call(fail, vec![wrap_io_error(e)]).wrap()],
}
}
}
pub type WriteManager<P> = IOManager<P, WriteCmd, (ExprInst, ExprInst)>;

6
src/systems/io/io.orc
View File

@@ -7,10 +7,10 @@ export const print := \text.\ok. (
(io::flush io::stdout
ok
(\e. panic "println threw on flush")
yield
\_. yield
)
(\e. panic "print threw on write")
yield
\_. yield
)
export const println := \line.\ok. (
@@ -21,7 +21,7 @@ export const readln := \ok. (
io::read_line io::stdin
ok
(\e. panic "readln threw")
yield
\_. yield
)
export module prelude (

9
src/systems/io/mod.rs
View File

@@ -1,6 +1,11 @@
//! System that allows Orchid to interact with trait objects of Rust's `Writer`
//! and with `BufReader`s of `Reader` trait objects
mod bindings;
mod facade;
// mod facade;
mod flow;
mod instances;
mod service;
pub use facade::{io_system, IOStream, IOSystem};
// pub use facade::{io_system, IOStream, IOSystem};
pub use service::{Service, Stream, StreamTable};

119
src/systems/io/service.rs Normal file
View File

@@ -0,0 +1,119 @@
#[allow(unused)] // for doc
use std::io::{BufReader, Read, Write};
use rust_embed::RustEmbed;
use trait_set::trait_set;
use super::bindings::io_bindings;
use super::flow::{IOCmd, IOCmdHandlePack};
use super::instances::{ReadCmd, Sink, Source, WriteCmd};
use crate::facade::{IntoSystem, System};
use crate::foreign::cps_box::{init_cps, CPSBox};
use crate::foreign::Atomic;
use crate::interpreter::HandlerTable;
use crate::pipeline::file_loader::embed_to_map;
use crate::sourcefile::{FileEntry, FileEntryKind, Import};
use crate::systems::codegen::call;
use crate::systems::scheduler::{SeqScheduler, SharedHandle};
use crate::Location;
/// A shared type for sinks and sources
pub enum Stream {
/// A Source, aka. a BufReader
Source(Source),
/// A Sink, aka. a Writer
Sink(Sink),
}
trait_set! {
/// The table of default streams to be overlain on the I/O module, typicially
/// stdin, stdout, stderr.
pub trait StreamTable<'a> = IntoIterator<Item = (&'a str, Stream)>
}
#[derive(RustEmbed)]
#[folder = "src/systems/io"]
#[prefix = "system/"]
#[include = "*.orc"]
struct IOEmbed;
/// A streaming I/O service for interacting with Rust's [Write] and [Read]
/// traits.
pub struct Service<'a, ST: IntoIterator<Item = (&'a str, Stream)>> {
scheduler: SeqScheduler,
global_streams: ST,
}
impl<'a, ST: IntoIterator<Item = (&'a str, Stream)>> Service<'a, ST> {
/// Construct a new instance of the service
pub fn new(scheduler: SeqScheduler, global_streams: ST) -> Self {
Self { scheduler, global_streams }
}
}
impl<'a, ST: IntoIterator<Item = (&'a str, Stream)>> IntoSystem<'static>
for Service<'a, ST>
{
fn into_system(self, i: &crate::Interner) -> crate::facade::System<'static> {
let scheduler = self.scheduler.clone();
let mut handlers = HandlerTable::new();
handlers.register(move |cps: &CPSBox<IOCmdHandlePack<ReadCmd>>| {
let (IOCmdHandlePack { cmd, handle }, succ, fail, tail) = cps.unpack3();
let (cmd, succ1, fail1) = (*cmd, succ.clone(), fail.clone());
let result = scheduler.schedule(
handle.clone(),
move |mut stream, cancel| {
let ret = cmd.execute(&mut stream, cancel);
(stream, ret)
},
move |stream, res, _cancel| (stream, res.dispatch(succ1, fail1)),
|stream| (stream, Vec::new()),
);
match result {
Ok(cancel) =>
Ok(call(tail.clone(), vec![init_cps(1, cancel).wrap()]).wrap()),
Err(e) => Ok(call(fail.clone(), vec![e.atom_exi()]).wrap()),
}
});
let scheduler = self.scheduler.clone();
handlers.register(move |cps: &CPSBox<IOCmdHandlePack<WriteCmd>>| {
let (IOCmdHandlePack { cmd, handle }, succ, fail, tail) = cps.unpack3();
let (cmd, succ1, fail1) = (cmd.clone(), succ.clone(), fail.clone());
let result = scheduler.schedule(
handle.clone(),
move |mut stream, cancel| {
let ret = cmd.execute(&mut stream, cancel);
(stream, ret)
},
move |stream, res, _cancel| (stream, res.dispatch(succ1, fail1)),
|stream| (stream, Vec::new()),
);
match result {
Ok(cancel) =>
Ok(call(tail.clone(), vec![init_cps(1, cancel).wrap()]).wrap()),
Err(e) => Ok(call(fail.clone(), vec![e.atom_exi()]).wrap()),
}
});
let streams = self.global_streams.into_iter().map(|(n, stream)| {
let handle = match stream {
Stream::Sink(sink) =>
Box::new(SharedHandle::wrap(sink)) as Box<dyn Atomic>,
Stream::Source(source) => Box::new(SharedHandle::wrap(source)),
};
(n, handle)
});
System {
handlers,
name: vec!["system".to_string(), "io".to_string()],
constants: io_bindings(i, streams).unwrap_tree(),
code: embed_to_map::<IOEmbed>(".orc", i),
prelude: vec![FileEntry {
locations: vec![Location::Unknown],
kind: FileEntryKind::Import(vec![Import {
location: Location::Unknown,
path: vec![i.i("system"), i.i("io"), i.i("prelude")],
name: None,
}]),
}],
}
}
}

8
src/systems/mod.rs
View File

@@ -1,13 +1,13 @@
//! Constants exposed to usercode by the interpreter
mod assertion_error;
mod asynch;
pub mod asynch;
pub mod cast_exprinst;
pub mod codegen;
mod io;
pub mod io;
mod runtime_error;
pub mod stl;
mod directfs;
pub mod scheduler;
pub use assertion_error::AssertionError;
pub use asynch::{AsynchConfig, InfiniteBlock, MessagePort};
pub use io::{io_system, IOStream, IOSystem};
pub use runtime_error::RuntimeError;

133
src/systems/scheduler/busy.rs Normal file
View File

@@ -0,0 +1,133 @@
use std::any::Any;
use std::collections::VecDeque;
use crate::interpreted::ExprInst;
use super::Canceller;
pub type SyncResult<T> = (T, Box<dyn Any + Send>);
pub type SyncOperation<T> =
Box<dyn FnOnce(T, Canceller) -> SyncResult<T> + Send>;
pub type SyncOpResultHandler<T> = Box<
dyn FnOnce(T, Box<dyn Any + Send>, Canceller) -> (T, Vec<ExprInst>),
>;
struct SyncQueueItem<T> {
cancelled: Canceller,
operation: SyncOperation<T>,
handler: SyncOpResultHandler<T>,
early_cancel: Box<dyn FnOnce(T) -> (T, Vec<ExprInst>)>,
}
pub enum NextItemReportKind<T> {
Free(T),
Next {
instance: T,
cancelled: Canceller,
operation: SyncOperation<T>,
rest: BusyState<T>,
},
Taken,
}
pub struct NextItemReport<T> {
pub kind: NextItemReportKind<T>,
pub events: Vec<ExprInst>,
}
pub struct BusyState<T> {
handler: SyncOpResultHandler<T>,
queue: VecDeque<SyncQueueItem<T>>,
seal: Option<Box<dyn FnOnce(T) -> Vec<ExprInst>>>,
}
impl<T> BusyState<T> {
pub fn new<U: 'static + Send>(
handler: impl FnOnce(T, U, Canceller) -> (T, Vec<ExprInst>) + 'static,
) -> Self {
BusyState {
handler: Box::new(|t, payload, cancel| {
let u = *payload
.downcast()
.expect("mismatched initial handler and operation");
handler(t, u, cancel)
}),
queue: VecDeque::new(),
seal: None,
}
}
/// Add a new operation to the queue. Returns Some if the operation was
/// successfully enqueued and None if the queue is already sealed.
pub fn enqueue<U: 'static + Send>(
&mut self,
operation: impl FnOnce(T, Canceller) -> (T, U) + Send + 'static,
handler: impl FnOnce(T, U, Canceller) -> (T, Vec<ExprInst>) + 'static,
early_cancel: impl FnOnce(T) -> (T, Vec<ExprInst>) + 'static,
) -> Option<Canceller> {
if self.seal.is_some() {
return None;
}
let cancelled = Canceller::new();
self.queue.push_back(SyncQueueItem {
cancelled: cancelled.clone(),
early_cancel: Box::new(early_cancel),
operation: Box::new(|t, c| {
let (t, r) = operation(t, c);
(t, Box::new(r))
}),
handler: Box::new(|t, u, c| {
let u = u.downcast().expect("mismatched handler and operation");
handler(t, *u, c)
}),
});
Some(cancelled)
}
pub fn seal(&mut self, recipient: impl FnOnce(T) -> Vec<ExprInst> + 'static) {
assert!(self.seal.is_none(), "Already sealed");
self.seal = Some(Box::new(recipient))
}
pub fn is_sealed(&self) -> bool {
self.seal.is_some()
}
pub fn rotate<U: Send + 'static>(
mut self,
instance: T,
result: U,
cancelled: Canceller,
) -> NextItemReport<T> {
let (mut instance, mut events) =
(self.handler)(instance, Box::new(result), cancelled);
let next_item = loop {
if let Some(candidate) = self.queue.pop_front() {
if candidate.cancelled.is_cancelled() {
let ret = (candidate.early_cancel)(instance);
instance = ret.0;
events.extend(ret.1.into_iter());
} else {
break candidate;
}
} else if let Some(seal) = self.seal.take() {
seal(instance);
let kind = NextItemReportKind::Taken;
return NextItemReport { events, kind };
} else {
let kind = NextItemReportKind::Free(instance);
return NextItemReport { events, kind };
}
};
self.handler = next_item.handler;
NextItemReport {
events,
kind: NextItemReportKind::Next {
instance,
cancelled: next_item.cancelled,
operation: next_item.operation,
rest: self,
},
}
}
}

32
src/systems/scheduler/canceller.rs Normal file
View File

@@ -0,0 +1,32 @@
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use crate::atomic_inert;
/// A single-fire thread-safe boolean flag with relaxed ordering
#[derive(Debug, Clone)]
pub struct Canceller(Arc<AtomicBool>);
atomic_inert!(Canceller, typestr = "a canceller");
impl Canceller {
/// Create a new canceller
pub fn new() -> Self {
Canceller(Arc::new(AtomicBool::new(false)))
}
/// Check whether the operation has been cancelled
pub fn is_cancelled(&self) -> bool {
self.0.load(Ordering::Relaxed)
}
/// Cancel the operation
pub fn cancel(&self) {
self.0.store(true, Ordering::Relaxed)
}
}
impl Default for Canceller {
fn default() -> Self {
Self::new()
}
}

10
src/systems/scheduler/mod.rs Normal file
View File

@@ -0,0 +1,10 @@
//! A generic utility to sequence long blocking mutations that require a mutable
//! reference to a shared resource.
mod busy;
mod system;
mod canceller;
mod take_and_drop;
pub use canceller::Canceller;
pub use system::{SealedOrTaken, SeqScheduler, SharedHandle, SharedState};

305
src/systems/scheduler/system.rs Normal file
View File

@@ -0,0 +1,305 @@
use std::any::{type_name, Any};
use std::cell::RefCell;
use std::fmt::Debug;
use std::rc::Rc;
use hashbrown::HashMap;
use itertools::Itertools;
use trait_set::trait_set;
use super::busy::{BusyState, NextItemReportKind};
use super::take_and_drop::{request, TakeAndDrop, TakeCmd};
use super::Canceller;
use crate::facade::{IntoSystem, System};
use crate::foreign::cps_box::CPSBox;
use crate::interpreted::ExprInst;
use crate::interpreter::HandlerTable;
use crate::systems::asynch::{AsynchSystem, MessagePort};
use crate::systems::stl::Boolean;
use crate::utils::thread_pool::ThreadPool;
use crate::utils::{take_with_output, unwrap_or, IdMap};
use crate::{atomic_inert, define_fn, ConstTree};
enum SharedResource<T> {
Free(T),
Busy(BusyState<T>),
Taken,
}
/// Possible states of a shared resource
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub enum SharedState {
/// The resource is ready to be used or taken
Free,
/// The resource is currently in use but operations can be asynchronously
/// scheduled on it
Busy,
/// The resource is currently in use and a consuming seal has already been
/// scheduled, therefore further operations cannot access it and it will
/// transition to [SharedState::Taken] as soon as the currently pending
/// operations finish or are cancelled.
Sealed,
/// The resource has been removed from this location.
Taken,
}
/// A shared handle for a resource of type `T` that can be used with a
/// [SeqScheduler] to execute mutating operations one by one in worker threads.
pub struct SharedHandle<T> {
state: Rc<RefCell<SharedResource<T>>>,
}
impl<T> SharedHandle<T> {
/// Wrap a value to be accessible to a [SeqScheduler].
pub fn wrap(t: T) -> Self {
Self { state: Rc::new(RefCell::new(SharedResource::Free(t))) }
}
/// Check the state of the handle
pub fn state(&self) -> SharedState {
match &*self.state.as_ref().borrow() {
SharedResource::Busy(b) if b.is_sealed() => SharedState::Sealed,
SharedResource::Busy(_) => SharedState::Busy,
SharedResource::Free(_) => SharedState::Free,
SharedResource::Taken => SharedState::Taken,
}
}
/// Remove the value from the handle if it's free. To interact with a handle
/// you probably want to use a [SeqScheduler], but sometimes this makes
/// sense as eg. an optimization. You can return the value after processing
/// via [SyncHandle::untake].
pub fn take(&self) -> Option<T> {
take_with_output(
&mut *self.state.as_ref().borrow_mut(),
|state| match state {
SharedResource::Free(t) => (SharedResource::Taken, Some(t)),
_ => (state, None),
},
)
}
/// Return the value to a handle that doesn't have one. The intended use case
/// is to return values synchronously after they have been removed with
/// [SyncHandle::untake].
pub fn untake(&self, value: T) -> Result<(), T> {
take_with_output(
&mut *self.state.as_ref().borrow_mut(),
|state| match state {
SharedResource::Taken => (SharedResource::Free(value), Ok(())),
_ => (state, Err(value)),
},
)
}
}
impl<T> Clone for SharedHandle<T> {
fn clone(&self) -> Self {
Self { state: self.state.clone() }
}
}
impl<T> Debug for SharedHandle<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SharedHandle")
.field("state", &self.state())
.field("type", &type_name::<T>())
.finish()
}
}
atomic_inert! {
SharedHandle(T),
typestr = "a shared handle",
request = |req: Box<dyn Any>, this: &SharedHandle<T>| request(this, req)
}
/// Error produced when an operation is scheduled or a seal placed on a resource
/// which is either already sealed or taken.
#[derive(Debug, Clone)]
pub struct SealedOrTaken;
atomic_inert!(
SealedOrTaken,
typestr = "a sealed-or-taken error for a shared resource"
);
define_fn! {
IsTakenError = |x| Ok(Boolean(SealedOrTaken::try_from(x).is_ok()).atom_cls())
}
trait_set! {
/// The part of processing a blocking I/O task that cannot be done on a remote
/// thread, eg. because it accesses other systems or Orchid code.
trait NonSendFn = FnOnce(Box<dyn Any>, SeqScheduler) -> Vec<ExprInst>;
}
struct SyncReply {
opid: u64,
data: Box<dyn Any + Send>,
}
struct CheshireCat {
pool: ThreadPool<Box<dyn FnOnce() + Send>>,
pending: RefCell<IdMap<Box<dyn NonSendFn>>>,
port: MessagePort,
}
/// A task scheduler that executes long blocking operations that have mutable
/// access to a shared one by one on a worker thread. The resources are
/// held in [SharedHandle]s
#[derive(Clone)]
pub struct SeqScheduler(Rc<CheshireCat>);
impl SeqScheduler {
/// Creates a new [SeqScheduler]. The new object is also kept alive by a
/// callback in the provided [AsynchSystem]. There should be at most one
pub fn new(asynch: &mut AsynchSystem) -> Self {
let this = Self(Rc::new(CheshireCat {
pending: RefCell::new(IdMap::new()),
pool: ThreadPool::new(),
port: asynch.get_port(),
}));
let this1 = this.clone();
// referenced by asynch, references this
asynch.register(move |res: Box<SyncReply>| {
let callback = this1.0.pending.borrow_mut().remove(res.opid).expect(
"Received reply for task we didn't start. This likely means that \
there are multiple SequencingContexts attached to the same \
AsynchSystem.",
);
callback(res.data, this1.clone())
});
this
}
/// Submit an action to be executed on a worker thread which can own the data
/// in the handle.
///
/// * handle - data to be transformed
/// * operation - long blocking mutation to execute off-thread.
/// * handler - process the results, talk to other systems, generate and run
/// Orchid code.
/// * early_cancel - clean up in case the task got cancelled before it was
/// scheduled. This is an optimization so that threads aren't spawned if a
/// large batch of tasks is scheduled and then cancelled.
pub fn schedule<T: Send + 'static, U: Send + 'static>(
&self,
handle: SharedHandle<T>,
operation: impl FnOnce(T, Canceller) -> (T, U) + Send + 'static,
handler: impl FnOnce(T, U, Canceller) -> (T, Vec<ExprInst>) + 'static,
early_cancel: impl FnOnce(T) -> (T, Vec<ExprInst>) + 'static,
) -> Result<Canceller, SealedOrTaken> {
take_with_output(&mut *handle.state.as_ref().borrow_mut(), {
let handle = handle.clone();
|state| {
match state {
SharedResource::Taken => (SharedResource::Taken, Err(SealedOrTaken)),
SharedResource::Busy(mut b) =>
match b.enqueue(operation, handler, early_cancel) {
Some(cancelled) => (SharedResource::Busy(b), Ok(cancelled)),
None => (SharedResource::Busy(b), Err(SealedOrTaken)),
},
SharedResource::Free(t) => {
let cancelled = Canceller::new();
drop(early_cancel); // cannot possibly be useful
self.submit(t, handle, cancelled.clone(), operation);
(SharedResource::Busy(BusyState::new(handler)), Ok(cancelled))
},
}
}
})
}
/// Schedule a function that will consume the value. After this the handle is
/// considered sealed and all [SeqScheduler::schedule] calls will fail.
pub fn seal<T>(
&self,
handle: SharedHandle<T>,
seal: impl FnOnce(T) -> Vec<ExprInst> + 'static,
) -> Result<Vec<ExprInst>, SealedOrTaken> {
take_with_output(&mut *handle.state.as_ref().borrow_mut(), |state| {
match state {
SharedResource::Busy(mut b) if !b.is_sealed() => {
b.seal(seal);
(SharedResource::Busy(b), Ok(Vec::new()))
},
SharedResource::Busy(_) => (state, Err(SealedOrTaken)),
SharedResource::Taken => (SharedResource::Taken, Err(SealedOrTaken)),
SharedResource::Free(t) => (SharedResource::Taken, Ok(seal(t))),
}
})
}
/// Asynchronously recursive function to schedule a new task for execution and
/// act upon its completion. The self-reference is passed into the callback
/// from the callback passed to the [AsynchSystem] so that if the task is
/// never resolved but the [AsynchSystem] through which the resolving event
/// would arrive is dropped this [SeqScheduler] is also dropped.
fn submit<T: Send + 'static, U: Send + 'static>(
&self,
t: T,
handle: SharedHandle<T>,
cancelled: Canceller,
operation: impl FnOnce(T, Canceller) -> (T, U) + Send + 'static,
) {
// referenced by self until run, references handle
let opid = self.0.pending.borrow_mut().insert(Box::new({
let cancelled = cancelled.clone();
move |data, this| {
let (t, u): (T, U) =
*data.downcast().expect("This is associated by ID");
let handle2 = handle.clone();
take_with_output(&mut *handle.state.as_ref().borrow_mut(), |state| {
let busy = unwrap_or! { state => SharedResource::Busy;
panic!("Handle with outstanding invocation must be busy")
};
let report = busy.rotate(t, u, cancelled);
match report.kind {
NextItemReportKind::Free(t) =>
(SharedResource::Free(t), report.events),
NextItemReportKind::Taken => (SharedResource::Taken, report.events),
NextItemReportKind::Next {
instance,
cancelled,
operation,
rest,
} => {
this.submit(instance, handle2, cancelled, operation);
(SharedResource::Busy(rest), report.events)
},
}
})
}
}));
let mut port = self.0.port.clone();
// referenced by thread until run, references port
self.0.pool.submit(Box::new(move || {
port.send(SyncReply { opid, data: Box::new(operation(t, cancelled)) })
}))
}
}
impl IntoSystem<'static> for SeqScheduler {
fn into_system(self, i: &crate::Interner) -> crate::facade::System<'static> {
let mut handlers = HandlerTable::new();
handlers.register(|cmd: &CPSBox<Canceller>| {
let (canceller, cont) = cmd.unpack1();
canceller.cancel();
Ok(cont.clone())
});
handlers.register(move |cmd: &CPSBox<TakeCmd>| {
let (TakeCmd(cb), cont) = cmd.unpack1();
cb(self.clone());
Ok(cont.clone())
});
System {
name: ["system", "scheduler"].into_iter().map_into().collect(),
prelude: Vec::new(),
code: HashMap::new(),
handlers,
constants: ConstTree::namespace(
[i.i("system"), i.i("scheduler")],
ConstTree::tree([
(i.i("is_taken_error"), ConstTree::xfn(IsTakenError)),
(i.i("take_and_drop"), ConstTree::xfn(TakeAndDrop)),
]),
)
.unwrap_tree(),
}
}
}

44
src/systems/scheduler/take_and_drop.rs Normal file
View File

@@ -0,0 +1,44 @@
use std::any::Any;
use std::fmt::Debug;
use std::rc::Rc;
use super::{SeqScheduler, SharedHandle};
use crate::foreign::cps_box::{init_cps, CPSBox};
use crate::foreign::Atom;
use crate::interpreted::Clause;
use crate::systems::AssertionError;
use crate::{define_fn, Primitive};
pub fn request<T: 'static>(
handle: &SharedHandle<T>,
request: Box<dyn Any>,
) -> Option<Box<dyn Any>> {
if request.downcast::<TakerRequest>().is_ok() {
let handle = handle.clone();
let cmd = TakeCmd(Rc::new(move |sch| {
let _ = sch.seal(handle.clone(), |_| Vec::new());
}));
return Some(Box::new(init_cps(1, cmd)))
}
None
}
pub struct TakerRequest;
#[derive(Clone)]
pub struct TakeCmd(pub Rc<dyn Fn(SeqScheduler)>);
impl Debug for TakeCmd {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "A command to drop a shared resource")
}
}
define_fn! {
pub TakeAndDrop = |x| x.inspect(|c| match c {
Clause::P(Primitive::Atom(Atom(atomic))) => {
let t = atomic.request(Box::new(TakerRequest))
.ok_or_else(|| AssertionError::ext(x.clone(), "a SharedHandle"))?;
let data: CPSBox<TakeCmd> = *t.downcast().expect("implied by request");
Ok(data.atom_cls())
},
_ => AssertionError::fail(x.clone(), "an atom"),
})
}

3
src/systems/stl/bin.rs
View File

@@ -4,7 +4,6 @@ use std::sync::Arc;
use itertools::Itertools;
use super::Boolean;
use crate::interpreted::ExprInst;
use crate::systems::cast_exprinst::with_uint;
use crate::systems::codegen::{orchid_opt, tuple};
use crate::systems::RuntimeError;
@@ -14,7 +13,7 @@ use crate::{atomic_inert, define_fn, ConstTree, Interner, Literal};
/// A block of binary data
#[derive(Clone, Hash, PartialEq, Eq)]
pub struct Binary(pub Arc<Vec<u8>>);
atomic_inert!(Binary, "a binary blob");
atomic_inert!(Binary, typestr = "a binary blob");
impl Debug for Binary {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

4
src/systems/stl/bool.rs
View File

@@ -1,14 +1,14 @@
use std::rc::Rc;
use crate::interner::Interner;
use crate::representations::interpreted::{Clause, ExprInst};
use crate::representations::interpreted::Clause;
use crate::systems::AssertionError;
use crate::{atomic_inert, define_fn, ConstTree, Literal, PathSet};
/// Booleans exposed to Orchid
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Boolean(pub bool);
atomic_inert!(Boolean, "a boolean");
atomic_inert!(Boolean, typestr = "a boolean");
impl From<bool> for Boolean {
fn from(value: bool) -> Self {

2
src/systems/stl/state.rs
View File

@@ -10,7 +10,7 @@ use crate::{atomic_inert, define_fn, ConstTree, Interner};
#[derive(Debug, Clone)]
pub struct State(Rc<RefCell<ExprInst>>);
atomic_inert!(State, "a state");
atomic_inert!(State, typestr = "a state");
#[derive(Debug, Clone)]
struct NewStateCmd;