New plans for macros

About to move them completely to stdlib

orchid-api/src/atom.rs

@@ -1,18 +1,24 @@
+use std::num::NonZeroU64;
+
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::ProjResult;
+use crate::error::OrcResult;
 use crate::expr::{Expr, ExprTicket};
 use crate::proto::{ExtHostReq, HostExtNotif, HostExtReq};
 use crate::system::SysId;
 
 pub type AtomData = Vec<u8>;
 
+/// Unique ID associated with atoms that have an identity
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct AtomId(pub NonZeroU64);
+
 /// An atom owned by an implied system. Usually used in responses from a system.
 /// This has the same semantics as [Atom] except in that the owner is implied.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub struct LocalAtom {
-  pub drop: bool,
+  pub drop: Option<AtomId>,
   pub data: AtomData,
 }
 impl LocalAtom {
@@ -25,10 +31,10 @@ impl LocalAtom {
 pub struct Atom {
   /// Instance ID of the system that created the atom
   pub owner: SysId,
-  /// Indicates whether the owner should be notified when this atom is dropped.
+  /// Indicates how the owner should be notified when this atom is dropped.
   /// Construction is always explicit and atoms are never cloned.
   ///
-  /// Atoms with `drop == false` are also known as trivial, they can be
+  /// Atoms with `drop == None` are also known as trivial, they can be
   /// duplicated and stored with no regard to expression lifetimes. NOTICE
   /// that this only applies to the atom. If it's referenced with an
   /// [ExprTicket], the ticket itself can still expire.
@@ -36,7 +42,7 @@ pub struct Atom {
   /// Notice also that the atoms still expire when the system is dropped, and
   /// are not portable across instances of the same system, so this doesn't
   /// imply that the atom is serializable.
-  pub drop: bool,
+  pub drop: Option<AtomId>,
   /// Data stored in the atom. This could be a key into a map, or the raw data
   /// of the atom if it isn't too big.
   pub data: AtomData,
@@ -60,6 +66,20 @@ impl Request for FinalCall {
   type Response = Expr;
 }
 
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct SerializeAtom(pub Atom);
+impl Request for SerializeAtom {
+  type Response = (Vec<u8>, Vec<ExprTicket>);
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct DeserAtom(pub SysId, pub Vec<u8>, pub Vec<ExprTicket>);
+impl Request for DeserAtom {
+  type Response = Atom;
+}
+
 /// Determine whether two atoms are identical for the purposes of macro
 /// application. If a given atom is never generated by macros or this relation
 /// is difficult to define, the module can return false
@@ -94,7 +114,7 @@ pub enum NextStep {
 #[extends(AtomReq, HostExtReq)]
 pub struct Command(pub Atom);
 impl Request for Command {
-  type Response = ProjResult<NextStep>;
+  type Response = OrcResult<NextStep>;
 }
 
 /// Notification that an atom is being dropped because its associated expression
@@ -102,7 +122,7 @@ impl Request for Command {
 /// flag is false.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(HostExtNotif)]
-pub struct AtomDrop(pub Atom);
+pub struct AtomDrop(pub SysId, pub AtomId);
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
@@ -122,6 +142,7 @@ pub enum AtomReq {
   Fwded(Fwded),
   Command(Command),
   AtomPrint(AtomPrint),
+  SerializeAtom(SerializeAtom),
 }
 impl AtomReq {
   /// Obtain the first [Atom] argument of the request. All requests in this
@@ -133,7 +154,8 @@ impl AtomReq {
       | Self::Command(Command(a))
       | Self::FinalCall(FinalCall(a, ..))
       | Self::Fwded(Fwded(a, ..))
-      | Self::AtomPrint(AtomPrint(a)) => a,
+      | Self::AtomPrint(AtomPrint(a))
+      | Self::SerializeAtom(SerializeAtom(a)) => a,
     }
   }
 }

orchid-api/src/error.rs

@@ -1,18 +1,16 @@
 use std::num::NonZeroU16;
 use std::sync::Arc;
 
-use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::Request;
+use orchid_api_derive::Coding;
 
 use crate::interner::TStr;
 use crate::location::Location;
-use crate::proto::ExtHostReq;
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-pub struct ProjErrId(pub NonZeroU16);
+pub struct ErrId(pub NonZeroU16);
 
 #[derive(Clone, Debug, Coding)]
-pub struct ProjErrLocation {
+pub struct ErrLocation {
   /// Description of the relation of this location to the error. If not used,
   /// set to empty string
   pub message: Arc<String>,
@@ -26,7 +24,7 @@ pub struct ProjErrLocation {
 /// For example, file reading produces result::err when the file doesn't exist,
 /// and a bottom if the file name isn't a string.
 #[derive(Clone, Debug, Coding)]
-pub struct ProjErr {
+pub struct OrcError {
   /// General description of the kind of error.
   pub description: TStr,
   /// Specific information about the exact error, preferably containing concrete
@@ -34,22 +32,8 @@ pub struct ProjErr {
   pub message: Arc<String>,
   /// Specific code fragments that have contributed to the emergence of the
   /// error.
-  pub locations: Vec<ProjErrLocation>,
+  pub locations: Vec<ErrLocation>,
 }
 
 /// If this is an [`Err`] then the [`Vec`] must not be empty.
-pub type ProjResult<T> = Result<T, Vec<ProjErr>>;
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ProjErrReq, ExtHostReq)]
-pub struct GetErrorDetails(pub ProjErrId);
-impl Request for GetErrorDetails {
-  type Response = ProjErr;
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostReq)]
-#[extendable]
-pub enum ProjErrReq {
-  GetErrorDetails(GetErrorDetails),
-}
+pub type OrcResult<T> = Result<T, Vec<OrcError>>;

orchid-api/src/expr.rs

@@ -3,8 +3,8 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::atom::{Atom, LocalAtom};
-use crate::error::ProjErr;
+use crate::atom::Atom;
+use crate::error::OrcError;
 use crate::interner::TStrv;
 use crate::location::Location;
 use crate::proto::{ExtHostNotif, ExtHostReq};
@@ -47,7 +47,7 @@ pub struct Release(pub SysId, pub ExprTicket);
 /// [Release].
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExprNotif, ExtHostNotif)]
-pub struct Relocate {
+pub struct Move {
   pub dec: SysId,
   pub inc: SysId,
   pub expr: ExprTicket,
@@ -77,7 +77,7 @@ pub enum Clause {
   /// Because the atom is newly constructed, it also must belong to this system.
   /// For convenience, [SysId::MAX] is also accepted as referring to this
   /// system.
-  NewAtom(LocalAtom),
+  NewAtom(Atom),
   /// An atom, specifically an atom that already exists. This form is only ever
   /// returned from [Inspect], and it's borrowed from the expression being
   /// inspected.
@@ -85,7 +85,7 @@ pub enum Clause {
   /// A reference to a constant
   Const(TStrv),
   /// A static runtime error.
-  Bottom(Vec<ProjErr>),
+  Bottom(Vec<OrcError>),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -120,5 +120,5 @@ pub enum ExprReq {
 pub enum ExprNotif {
   Acquire(Acquire),
   Release(Release),
-  Relocate(Relocate),
+  Move(Move),
 }

orchid-api/src/interner.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::proto::{ExtHostNotif, ExtHostReq, HostExtReq};
+use crate::proto::{ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
@@ -80,16 +80,6 @@ impl Request for Sweep {
   type Response = Retained;
 }
 
-/// A notification from the extension to the host, that the extension would benefit from a sweep
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostNotif)]
-pub struct AdviseSweep(SweepReason);
-
-#[derive(Clone, Debug, Coding)]
-pub enum SweepReason {
-  None
-}
-
 /// List of keys in this replica that couldn't be sweeped because local
 /// datastructures reference their value.
 #[derive(Clone, Debug, Coding)]

orchid-api/src/lib.rs

@@ -1,11 +1,35 @@
-pub mod atom;
-pub mod error;
-pub mod expr;
-pub mod interner;
-pub mod location;
-pub mod parser;
-pub mod proto;
-pub mod system;
-pub mod tree;
-pub mod vfs;
-pub mod logging;
+mod atom;
+pub use atom::{
+  Atom, AtomData, AtomDrop, AtomId, AtomPrint, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwd,
+  Fwded, LocalAtom, NextStep, DeserAtom, SerializeAtom
+};
+mod error;
+pub use error::{ErrId, ErrLocation, OrcError, OrcResult};
+mod expr;
+pub use expr::{
+  Acquire, Clause, Details, Expr, ExprNotif, ExprReq, ExprTicket, Inspect, Move, Release,
+};
+mod interner;
+pub use interner::{
+  ExternStr, ExternStrv, IntReq, InternStr, InternStrv, Retained, Sweep, TStr, TStrv,
+};
+mod location;
+pub use location::{CodeGenInfo, Location, SourceRange};
+mod logging;
+pub use logging::{Log, LogStrategy};
+mod parser;
+pub use parser::{CharFilter, LexExpr, LexedExpr, ParsId, ParseLine, ParserReq, SubLex, SubLexed};
+mod proto;
+pub use proto::{
+  ExtHostChannel, ExtHostNotif, ExtHostReq, ExtMsgSet, ExtensionHeader, HostExtChannel,
+  HostExtNotif, HostExtReq, HostHeader, HostMsgSet, Ping,
+};
+mod system;
+pub use system::{SysReq, NewSystem, SysDeclId, SysId, SystemDecl, SystemDrop, SystemInst};
+mod tree;
+pub use tree::{
+  CompName, GetMember, Item, ItemKind, Macro, Member, MemberKind, Module, Paren, Placeholder,
+  PlaceholderKind, Token, TokenTree, TreeId, TreeTicket,
+};
+mod vfs;
+pub use vfs::{EagerVfs, GetVfs, Loaded, VfsId, VfsRead, VfsReq};

orchid-api/src/logging.rs

@@ -5,9 +5,9 @@ use crate::proto::ExtHostNotif;
 #[derive(Clone, Debug, Coding)]
 pub enum LogStrategy {
   StdErr,
-  File(String)
+  File(String),
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostNotif)]
-pub struct Log(pub String);
+pub struct Log(pub String);

orchid-api/src/parser.rs

@@ -4,7 +4,7 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::ProjResult;
+use crate::error::OrcResult;
 use crate::interner::TStr;
 use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
@@ -24,6 +24,7 @@ pub struct CharFilter(pub Vec<RangeInclusive<char>>);
 #[extendable]
 pub enum ParserReq {
   LexExpr(LexExpr),
+  ParseLine(ParseLine),
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
@@ -35,7 +36,7 @@ pub struct LexExpr {
   pub pos: u32,
 }
 impl Request for LexExpr {
-  type Response = Option<ProjResult<LexedExpr>>;
+  type Response = Option<OrcResult<LexedExpr>>;
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -60,11 +61,12 @@ pub struct SubLexed {
   pub ticket: TreeTicket,
 }
 
-#[derive(Clone, Debug, Coding)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ParserReq, HostExtReq)]
 pub struct ParseLine {
   pub sys: SysId,
   pub line: Vec<TokenTree>,
 }
 impl Request for ParseLine {
-  type Response = Vec<TokenTree>;
+  type Response = OrcResult<Vec<TokenTree>>;
 }

orchid-api/src/proto.rs

@@ -27,7 +27,8 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::{atom, error, expr, interner, logging::{self, LogStrategy}, parser, system, tree, vfs};
+use crate::logging::{self, LogStrategy};
+use crate::{atom, expr, interner, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader {
@@ -48,17 +49,19 @@ impl Encode for HostHeader {
 
 static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
 pub struct ExtensionHeader {
+  pub name: String,
   pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
   fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, EXT_INTRO);
-    Self { systems: Vec::decode(read) }
+    Self { name: String::decode(read), systems: Vec::decode(read) }
   }
 }
 impl Encode for ExtensionHeader {
   fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     write_exact(write, EXT_INTRO);
+    self.name.encode(write);
     self.systems.encode(write)
   }
 }
@@ -78,7 +81,6 @@ pub enum ExtHostReq {
   Fwd(atom::Fwd),
   ExprReq(expr::ExprReq),
   SubLex(parser::SubLex),
-  ProjErrReq(error::ProjErrReq),
 }
 
 /// Notifications sent from the extension to the host
@@ -87,7 +89,6 @@ pub enum ExtHostReq {
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
-  AdviseSweep(interner::AdviseSweep),
   Log(logging::Log),
 }
 
@@ -102,9 +103,10 @@ impl Channel for ExtHostChannel {
 #[extendable]
 pub enum HostExtReq {
   Ping(Ping),
-  NewSystem(system::NewSystem),
+  SysReq(system::SysReq),
   Sweep(interner::Sweep),
   AtomReq(atom::AtomReq),
+  DeserAtom(atom::DeserAtom),
   ParserReq(parser::ParserReq),
   GetMember(tree::GetMember),
   VfsReq(vfs::VfsReq),
@@ -143,35 +145,35 @@ impl MsgSet for HostMsgSet {
 
 #[cfg(test)]
 mod tests {
-    use ordered_float::NotNan;
-    use system::{SysDeclId, SystemDecl};
+  use orchid_api_traits::enc_vec;
+use ordered_float::NotNan;
+  use system::{SysDeclId, SystemDecl};
 
-    use super::*;
+  use super::*;
 
-    #[test]
-    fn host_header_enc() {
-      let hh = HostHeader { log_strategy: LogStrategy::File("SomeFile".to_string()) };
-      let mut enc = &hh.enc_vec()[..];
-      eprintln!("Encoded to {enc:?}");
-      HostHeader::decode(&mut enc);
-      assert_eq!(enc, []);
-    }
+  #[test]
+  fn host_header_enc() {
+    let hh = HostHeader { log_strategy: LogStrategy::File("SomeFile".to_string()) };
+    let mut enc = &enc_vec(&hh)[..];
+    eprintln!("Encoded to {enc:?}");
+    HostHeader::decode(&mut enc);
+    assert_eq!(enc, []);
+  }
 
-    #[test]
-    fn ext_header_enc() {
-      let eh = ExtensionHeader {
-        systems: vec![
-          SystemDecl {
-            id: SysDeclId(1.try_into().unwrap()),
-            name: "misc".to_string(),
-            depends: vec![ "std".to_string() ],
-            priority: NotNan::new(1f64).unwrap()
-          }
-        ]
-      };
-      let mut enc = &eh.enc_vec()[..];
-      eprintln!("Encoded to {enc:?}");
-      ExtensionHeader::decode(&mut enc);
-      assert_eq!(enc, [])
-    }
-}
+  #[test]
+  fn ext_header_enc() {
+    let eh = ExtensionHeader {
+      name: "my_extension".to_string(),
+      systems: vec![SystemDecl {
+        id: SysDeclId(1.try_into().unwrap()),
+        name: "misc".to_string(),
+        depends: vec!["std".to_string()],
+        priority: NotNan::new(1f64).unwrap(),
+      }],
+    };
+    let mut enc = &enc_vec(&eh)[..];
+    eprintln!("Encoded to {enc:?}");
+    ExtensionHeader::decode(&mut enc);
+    assert_eq!(enc, [])
+  }
+}

orchid-api/src/system.rs

@@ -18,8 +18,8 @@ pub struct SysDeclId(pub NonZeroU16);
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct SysId(pub NonZeroU16);
 
-/// Details about a system provided by this library. This is included in the extension header,
-/// so it cannot rely on the interner. 
+/// Details about a system provided by this library. This is included in the
+/// extension header, so it cannot rely on the interner.
 #[derive(Debug, Clone, Coding)]
 pub struct SystemDecl {
   /// ID of the system, unique within the library
@@ -44,7 +44,7 @@ pub struct SystemDecl {
 /// essential that any resource associated with a system finds its system by the
 /// ID in a global map.
 #[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtReq)]
+#[extends(SysReq, HostExtReq)]
 pub struct NewSystem {
   /// ID of the system
   pub system: SysDeclId,
@@ -64,10 +64,17 @@ pub struct SystemInst {
   /// can process. The lexer will notify this system if it encounters one of
   /// these characters.9
   pub lex_filter: CharFilter,
-  pub parses_lines: Vec<TStr>,
+  pub line_types: Vec<TStr>,
   pub const_root: HashMap<TStr, MemberKind>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtNotif)]
 pub struct SystemDrop(pub SysId);
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+#[extendable]
+pub enum SysReq {
+  NewSystem(NewSystem),
+}

orchid-api/src/tree.rs

@@ -1,18 +1,17 @@
-use crate::interner::TStrv;
-use crate::location::Location;
 use std::num::NonZeroU64;
 use std::ops::Range;
+use std::sync::Arc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::atom::LocalAtom;
-use crate::error::ProjErr;
-use crate::expr::Expr;
-use crate::interner::TStr;
+use crate::error::OrcError;
+use crate::interner::{TStr, TStrv};
+use crate::location::Location;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
+use crate::{Atom, Expr};
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
 /// the lexer can include it in its output or discard it by implication.
@@ -46,12 +45,14 @@ pub enum Token {
   /// line parser output
   Ph(Placeholder),
   /// A new atom
-  Atom(LocalAtom),
+  Atom(Atom),
   /// Anchor to insert a subtree
   Slot(TreeTicket),
   /// A static compile-time error returned by failing lexers if
   /// the rest of the source is likely still meaningful
-  Bottom(Vec<ProjErr>),
+  Bottom(Vec<OrcError>),
+  /// A comment
+  Comment(Arc<String>),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -84,10 +85,10 @@ pub struct Macro {
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct TreeId(pub NonZeroU64);
 
-
 #[derive(Clone, Debug, Coding)]
 pub struct Item {
   pub location: Location,
+  pub comments: Vec<(Arc<String>, Location)>,
   pub kind: ItemKind,
 }
 
@@ -95,13 +96,15 @@ pub struct Item {
 pub enum ItemKind {
   Member(Member),
   Raw(Vec<TokenTree>),
+  Export(TStr),
   Rule(Macro),
+  Import(CompName),
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct Member {
   pub name: TStr,
-  pub public: bool,
+  pub exported: bool,
   pub kind: MemberKind,
 }
 
@@ -118,6 +121,13 @@ pub struct Module {
   pub items: Vec<Item>,
 }
 
+#[derive(Clone, Debug, Coding)]
+pub struct CompName {
+  pub path: Vec<TStr>,
+  pub name: Option<TStr>,
+  pub location: Location,
+}
+
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct GetMember(pub SysId, pub TreeId);

orchid-api/src/vfs.rs

@@ -4,7 +4,7 @@ use std::num::NonZeroU16;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::ProjResult;
+use crate::error::OrcResult;
 use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
@@ -22,7 +22,7 @@ pub enum Loaded {
 #[extends(VfsReq, HostExtReq)]
 pub struct VfsRead(pub SysId, pub VfsId, pub Vec<TStr>);
 impl Request for VfsRead {
-  type Response = ProjResult<Loaded>;
+  type Response = OrcResult<Loaded>;
 }
 
 #[derive(Clone, Debug, Coding)]