in midst of refactor

orchid-api/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "orchid-api"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+ordered-float = "4.2.0"
+orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
+orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
+derive_more = "0.99.17"

orchid-api/src/atom.rs

@@ -0,0 +1,91 @@
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::expr::{Expr, ExprTicket};
+use crate::proto::{ExtHostReq, HostExtNotif, HostExtReq};
+use crate::system::SysId;
+
+pub type AtomData = Vec<u8>;
+
+/// An atom representation that can be serialized and sent around. Atoms
+/// represent the smallest increment of work.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct Atom {
+  /// Instance ID of the system that created the atom
+  pub owner: SysId,
+  /// Indicates whether the owner should be notified when this atom is dropped.
+  /// Construction is always explicit and atoms are never cloned.
+  ///
+  /// Atoms with `drop == false` are also known as trivial, they can be
+  /// duplicated and stored with no regard to expression lifetimes. NOTICE
+  /// that this only applies to the atom. If it's referenced with an
+  /// [ExprTicket], the ticket itself can still expire.
+  ///
+  /// Notice also that the atoms still expire when the system is dropped, and
+  /// are not portable across instances of the same system, so this doesn't
+  /// imply that the atom is serializable.
+  pub drop: bool,
+  /// Data stored in the atom. This could be a key into a map, or the raw data
+  /// of the atom if it isn't too big.
+  pub data: AtomData,
+}
+
+/// Attempt to apply an atom as a function to an expression
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct CallRef(pub Atom, pub ExprTicket);
+impl Request for CallRef {
+  type Response = Expr;
+}
+
+/// Attempt to apply an atom as a function, consuming the atom and enabling the
+/// library to reuse its datastructures rather than duplicating them. This is an
+/// optimization over [CallRef] followed by [AtomDrop].
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct FinalCall(pub Atom, pub ExprTicket);
+impl Request for FinalCall {
+  type Response = Expr;
+}
+
+/// Determine whether two atoms are identical for the purposes of macro
+/// application. If a given atom is never generated by macros or this relation
+/// is difficult to define, the module can return false
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct AtomSame(pub Atom, pub Atom);
+impl Request for AtomSame {
+  type Response = bool;
+}
+
+/// A request blindly routed to the system that provides an atom.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct Fwded(pub Atom, pub Vec<u8>);
+impl Request for Fwded {
+  type Response = Vec<u8>;
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct Fwd(pub Atom, pub Vec<u8>);
+impl Request for Fwd {
+  type Response = Vec<u8>;
+}
+
+/// Notification that an atom is being dropped because its associated expression
+/// isn't referenced anywhere. This should have no effect if the atom's `drop`
+/// flag is false.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(HostExtNotif)]
+pub struct AtomDrop(pub Atom);
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+#[extendable]
+pub enum AtomReq {
+  CallRef(CallRef),
+  FinalCall(FinalCall),
+  AtomSame(AtomSame),
+  Fwded(Fwded),
+}

orchid-api/src/error.rs

@@ -0,0 +1 @@
+pub type ProjErrId = u16;

orchid-api/src/expr.rs

@@ -0,0 +1,99 @@
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::atom::Atom;
+use crate::intern::{TStr, TStrv};
+use crate::location::Location;
+use crate::proto::ExtHostReq;
+use crate::system::SysId;
+
+/// An arbitrary ID associated with an expression on the host side. Incoming
+/// tickets always come with some lifetime guarantee, which can be extended with
+/// [AcquireExpr].
+///
+/// The ID is globally unique within its lifetime, but may be reused.
+pub type ExprTicket = u64;
+
+/// Acquire a strong reference to an expression. This keeps it alive until a
+/// corresponding [Release] is emitted. The number of times a system has
+/// acquired an expression is counted, and it is the system's responsibility to
+/// ensure that acquires and releases pair up. Behaviour in case of a
+/// superfluous free is not defined.
+///
+/// Some contexts may specify that an ingress [ExprTicket] is owned, this means
+/// that acquiring it is not necessary.
+///
+/// This can be called with a foreign system to signal that an owned reference
+/// is being passed, though [Relocate] may be a better fit.
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct Acquire(pub SysId, pub ExprTicket);
+
+/// Release a reference either previously acquired through either [Acquire]
+/// or by receiving an owned reference. The number of times a system has
+/// acquired an expression is counted, and it is the system's responsibility to
+/// ensure that acquires and releases pair up. Behaviour in case of excessive
+/// freeing is not defined.
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct Release(pub SysId, pub ExprTicket);
+
+/// Decrement the reference count for one system and increment it for another,
+/// to indicate passing an owned reference. Equivalent to [Acquire] followed by
+/// [Release].
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct Relocate {
+  pub dec: SysId,
+  pub inc: SysId,
+  pub expr: ExprTicket,
+}
+
+/// A description of a new expression. It is used as the return value of
+/// [crate::atom::Call] or [crate::atom::CallRef], or a constant in the
+/// [crate::tree::Tree].
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub enum Clause {
+  /// Apply the lhs as a function to the rhs
+  Call(Box<Expr>, Box<Expr>),
+  /// Lambda function. The number operates as an argument name
+  Lambda(TStr, Box<Expr>),
+  /// Binds the argument passed to the lambda with the same ID in the same
+  /// template
+  Arg(TStr),
+  /// Insert the specified host-expression in the template here. When the clause
+  /// is used in the const tree, this variant is forbidden.
+  Slot(ExprTicket),
+  /// The lhs must be fully processed before the rhs can be processed.
+  /// Equivalent to Haskell's function of the same name
+  Seq(Box<Expr>, Box<Expr>),
+  /// Insert an atom in the tree. When the clause is used in the const tree, the
+  /// atom must be trivial.
+  Atom(Atom),
+  /// A reference to a constant
+  Const(TStrv),
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct Expr {
+  pub clause: Clause,
+  pub location: Location
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(ExprReq, ExtHostReq)]
+pub struct Inspect(pub ExprTicket);
+impl Request for Inspect {
+  type Response = Clause;
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+#[extendable]
+pub enum ExprReq {
+  Inspect(Inspect),
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding)]
+pub enum ExprNotif {
+  Acquire(Acquire),
+  Release(Release),
+  Relocate(Relocate),
+}

orchid-api/src/fs.rs

@@ -0,0 +1,16 @@
+use orchid_api_derive::Coding;
+use orchid_api_traits::Request;
+
+pub type FsId = u16;
+
+#[derive(Clone, Debug, Coding)]
+pub enum Loaded {
+  Code(String),
+  Collection(Vec<String>),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct FsRead(pub Vec<String>);
+impl Request for FsRead {
+  type Response = Result<Loaded, ()>;
+}

orchid-api/src/intern.rs

@@ -0,0 +1,91 @@
+use std::num::NonZeroU64;
+use std::sync::Arc;
+
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::proto::{ExtHostReq, HostExtReq};
+
+/// Intern requests sent by the replica to the master. These requests are
+/// repeatable.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+#[extendable]
+pub enum IntReq {
+  InternStr(InternStr),
+  InternStrv(InternStrv),
+  ExternStr(ExternStr),
+  ExternStrv(ExternStrv),
+}
+
+/// replica -> master to intern a string on the master.
+///
+/// Repeatable.
+///
+/// See [IntReq]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(IntReq, ExtHostReq)]
+pub struct InternStr(pub Arc<String>);
+impl Request for InternStr {
+  type Response = TStr;
+}
+
+/// replica -> master to find the interned string corresponding to a key.
+///
+/// Repeatable.
+///
+/// See [IntReq]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(IntReq, ExtHostReq)]
+pub struct ExternStr(pub TStr);
+impl Request for ExternStr {
+  type Response = Arc<String>;
+}
+/// replica -> master to intern a vector of interned strings
+///
+/// Repeatable.
+///
+/// See [IntReq]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(IntReq, ExtHostReq)]
+pub struct InternStrv(pub Arc<Vec<TStr>>);
+impl Request for InternStrv {
+  type Response = TStrv;
+}
+/// replica -> master to find the vector of interned strings corresponding to a
+/// token
+///
+/// Repeatable.
+///
+/// See [IntReq]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(IntReq, ExtHostReq)]
+pub struct ExternStrv(pub TStrv);
+impl Request for ExternStrv {
+  type Response = Arc<Vec<TStr>>;
+}
+
+/// A substitute for an interned string in serialized datastructures.
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct TStr(pub NonZeroU64);
+
+/// A substitute for an interned string sequence in serialized datastructures.
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct TStrv(pub NonZeroU64);
+
+/// A request to sweep the replica. The master will not be sweeped until all
+/// replicas respond, as it must retain everything the replicas retained
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct Sweep;
+impl Request for Sweep {
+  type Response = Retained;
+}
+
+/// List of keys in this replica that couldn't be sweeped because local
+/// datastructures reference their value.
+#[derive(Clone, Debug, Coding)]
+pub struct Retained {
+  pub strings: Vec<TStr>,
+  pub vecs: Vec<TStrv>,
+}

orchid-api/src/lib.rs

@@ -0,0 +1,10 @@
+pub mod atom;
+pub mod error;
+pub mod expr;
+pub mod fs;
+pub mod intern;
+pub mod location;
+pub mod proto;
+pub mod system;
+pub mod tree;
+pub mod parser;

orchid-api/src/location.rs

@@ -0,0 +1,24 @@
+use std::ops::Range;
+
+use orchid_api_derive::Coding;
+
+use crate::intern::TStrv;
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub enum Location {
+  None,
+  Gen(CodeGenInfo),
+  Range(SourceRange),
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct SourceRange {
+  pub path: TStrv,
+  pub range: Range<u32>,
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct CodeGenInfo {
+  pub generator: String,
+  pub details: String,
+}

orchid-api/src/parser.rs

@@ -0,0 +1,55 @@
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::intern::TStr;
+use crate::proto::{ExtHostReq, HostExtReq};
+use crate::system::SysId;
+use crate::tree::TokenTree;
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+#[extendable]
+pub enum ParserReq {
+  MkLexer(MkLexer),
+  Lex(Lex),
+}
+
+pub type LexerId = u16;
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ParserReq, HostExtReq)]
+pub struct MkLexer(pub SysId, pub TStr);
+impl Request for MkLexer {
+  type Response = LexerId;
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ParserReq, HostExtReq)]
+pub struct Lex {
+  pub parser: LexerId,
+  pub next: char,
+  pub pos: u32,
+}
+impl Request for Lex {
+  type Response = Option<LexResult>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct LexResult {
+  pub consumed: u32,
+  pub data: TokenTree,
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct SubLex {
+  pub lexer: LexerId,
+  pub pos: u32,
+}
+impl Request for SubLex {
+  type Response = SubLex;
+}
+
+pub struct ParseLine {
+
+}

orchid-api/src/proto.rs

@@ -0,0 +1,125 @@
+//! Basic messages of the Orchid extension API.
+//!
+//! The protocol is defined over a byte stream, normally the stdin/stdout of the
+//! extension. The implementations of [Coding] in this library are considered
+//! normative. Any breaking change here or in the default implementations of
+//! [Coding] must also increment the version number in the intro strings.
+//!
+//! 3 different kinds of messages are recognized; request, response, and
+//! notification. There are no general ordering guarantees about these, multiple
+//! requests, even requests of the same type may be sent concurrently, unless
+//! otherwise specified in the request's definition.
+//!
+//! Each message begins with a u32 length, followed by that many bytes of
+//! message content. The first byte of the content is a u64 combined request ID
+//! and discriminator, D.
+//!
+//! - If D = 0, the rest of the content is a notification.
+//! - If 0 < D < 2^63, it is a request with identifier D.
+//! - If 2^63 <= D, it is a response to request identifier !D.
+//!
+//! The order of both notifications and requests sent from the same thread must
+//! be preserved. Toolkits must ensure that the client code is able to observe
+//! the ordering of messages.
+
+use std::io::{Read, Write};
+
+use derive_more::{From, TryInto};
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::{read_exact, write_exact, Decode, Encode, MsgSet, Request};
+
+use crate::{atom, expr, intern, parser, system, tree};
+
+static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
+pub struct HostHeader;
+impl Decode for HostHeader {
+  fn decode<R: Read>(read: &mut R) -> Self {
+    read_exact(read, HOST_INTRO);
+    Self
+  }
+}
+impl Encode for HostHeader {
+  fn encode<W: Write>(&self, write: &mut W) { write_exact(write, HOST_INTRO) }
+}
+
+static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
+pub struct ExtensionHeader {
+  pub systems: Vec<system::SystemDecl>,
+}
+impl Decode for ExtensionHeader {
+  fn decode<R: Read>(read: &mut R) -> Self {
+    read_exact(read, EXT_INTRO);
+    Self { systems: Vec::decode(read) }
+  }
+}
+impl Encode for ExtensionHeader {
+  fn encode<W: Write>(&self, write: &mut W) {
+    write_exact(write, EXT_INTRO);
+    self.systems.encode(write)
+  }
+}
+
+#[derive(Clone, Debug, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct Ping;
+impl Request for Ping {
+  type Response = ();
+}
+
+/// Requests running from the extension to the host
+#[derive(Clone, Coding, Hierarchy)]
+#[extendable]
+pub enum ExtHostReq {
+  Ping(Ping),
+  IntReq(intern::IntReq),
+  Fwd(atom::Fwd),
+  ExprReq(expr::ExprReq),
+  SubLex(parser::SubLex),
+}
+
+/// Notifications sent from the extension to the host
+#[derive(Coding, From, TryInto)]
+#[allow(clippy::enum_variant_names)]
+pub enum ExtHostNotif {
+  Expr(expr::ExprNotif),
+}
+
+/// Requests running from the host to the extension
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extendable]
+pub enum HostExtReq {
+  Ping(Ping),
+  NewSystem(system::NewSystem),
+  Sweep(intern::Sweep),
+  AtomReq(atom::AtomReq),
+  ParserReq(parser::ParserReq),
+  GetConstTree(tree::GetConstTree),
+}
+
+/// Notifications sent from the host to the extension
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extendable]
+pub enum HostExtNotif {
+  SystemDrop(system::SystemDrop),
+  AtomDrop(atom::AtomDrop),
+  /// The host can assume that after this notif is sent, a correctly written
+  /// extension will eventually exit.
+  Exit,
+}
+
+/// Message set viewed from the extension's perspective
+pub struct ExtMsgSet;
+impl MsgSet for ExtMsgSet {
+  type InNot = HostExtNotif;
+  type InReq = HostExtReq;
+  type OutNot = ExtHostNotif;
+  type OutReq = ExtHostReq;
+}
+
+/// Message Set viewed from the host's perspective
+pub struct HostMsgSet;
+impl MsgSet for HostMsgSet {
+  type InNot = ExtHostNotif;
+  type InReq = ExtHostReq;
+  type OutNot = HostExtNotif;
+  type OutReq = HostExtReq;
+}

orchid-api/src/system.rs

@@ -0,0 +1,54 @@
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+use ordered_float::NotNan;
+
+use crate::proto::{HostExtNotif, HostExtReq};
+
+/// ID of a system type
+pub type SysDeclId = u16;
+
+/// ID of a system instance
+pub type SysId = u16;
+
+/// Details about a system provided by this library
+#[derive(Coding)]
+pub struct SystemDecl {
+  /// ID of the system, unique within the library
+  pub id: SysDeclId,
+  /// This can be depended upon. Exactly one of each kind will be loaded
+  pub name: String,
+  /// If multiple instances of a system are found, the highest priority will be
+  /// used. This can be used for version counting, but also for fallbacks if a
+  /// negative number is found.
+  ///
+  /// Systems cannot depend on specific versions and older versions of systems
+  /// are never loaded. Compatibility can be determined on a per-system basis
+  /// through an algorithm chosen by the provider.
+  pub priority: NotNan<f64>,
+  /// List of systems needed for this one to work correctly. These will be
+  /// looked up, and an error produced if they aren't found.
+  pub depends: Vec<String>,
+}
+
+/// Host -> extension; instantiate a system according to its [SystemDecl].
+/// Multiple instances of a system may exist in the same address space, so it's
+/// essential that any resource associated with a system finds its system by the
+/// ID in a global map.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct NewSystem {
+  /// ID of the system
+  pub system: SysDeclId,
+  /// ID of the system instance, unique for the host
+  pub id: SysId,
+  /// Instance IDs for dependencies, in the order that the names appear in the
+  /// declaration
+  pub depends: Vec<SysId>,
+}
+impl Request for NewSystem {
+  type Response = ();
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtNotif)]
+pub struct SystemDrop(pub SysId);

orchid-api/src/tree.rs

@@ -0,0 +1,76 @@
+use std::collections::HashMap;
+
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+use ordered_float::NotNan;
+
+use crate::atom::Atom;
+use crate::expr::Expr;
+use crate::intern::TStr;
+use crate::location::SourceRange;
+use crate::proto::HostExtReq;
+use crate::system::SysId;
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct TokenTree {
+  token: Token,
+  location: SourceRange,
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub enum Token {
+  /// Lambda function. The number operates as an argument name
+  Lambda(TStr, Vec<TokenTree>),
+  Name(Vec<TStr>),
+  S(Paren, Vec<TokenTree>),
+  /// A placeholder in a macro. This variant is forbidden everywhere outside
+  /// line parser output
+  Ph(Placeholder),
+  Atom(Atom),
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct Placeholder {
+  name: TStr,
+  kind: PlaceholderKind,
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub enum PlaceholderKind {
+  Scalar,
+  Name,
+  Vector { nonzero: bool, priority: u8 },
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub enum Paren {
+  Round,
+  Square,
+  Curly,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct MacroRule {
+  pub pattern: Vec<TokenTree>,
+  pub priority: NotNan<f64>,
+  pub template: Vec<TokenTree>,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum Tree {
+  Const(Expr),
+  Mod(TreeModule),
+  Rule(MacroRule),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct TreeModule {
+  pub children: HashMap<String, Tree>,
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct GetConstTree(pub SysId);
+impl Request for GetConstTree {
+  type Response = TreeModule;
+}