Finally figured out how I want atoms to work

orchid-api/src/atom.rs

@@ -7,6 +7,14 @@ use crate::system::SysId;
 
 pub type AtomData = Vec<u8>;
 
+/// An atom owned by an implied system. Usually used in responses from a system.
+/// This has the same semantics as [Atom] except in that the owner is implied.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct LocalAtom {
+  pub drop: bool,
+  pub data: AtomData,
+}
+
 /// An atom representation that can be serialized and sent around. Atoms
 /// represent the smallest increment of work.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]

orchid-api/src/error.rs

@@ -1 +1,35 @@
+use orchid_api_derive::Coding;
+
+use crate::intern::TStr;
+use crate::location::Location;
+
 pub type ProjErrId = u16;
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct ProjErrLocation {
+  /// Description of the relation of this location to the error. If not used,
+  /// set to empty string
+  message: String,
+  /// Location in code where the error emerged. This is usually [Location::Gen].
+  location: Location,
+}
+
+/// Programming errors raised by extensions. At runtime these produce the
+/// equivalent of a Haskell bottom. Note that runtime errors produced by
+/// fallible operations should return an Orchid result and not a bottom.
+/// For example, file reading produces result::err when the file doesn't exist,
+/// and a bottom if the file name isn't a string.
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+pub struct ProjErr {
+  /// General description of the kind of error.
+  description: TStr,
+  /// Specific information about the exact error, preferably containing concrete
+  /// values.
+  message: String,
+  /// Specific code fragments that have contributed to the emergence of the
+  /// error.
+  locations: Vec<ProjErrLocation>,
+}
+
+/// If this is an [`Err`] then the [`Vec`] must not be empty.
+pub type ProjResult<T> = Result<T, Vec<ProjErr>>;

orchid-api/src/expr.rs

@@ -1,7 +1,7 @@
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::atom::Atom;
+use crate::atom::LocalAtom;
 use crate::intern::{TStr, TStrv};
 use crate::location::Location;
 use crate::proto::{ExtHostNotif, ExtHostReq};
@@ -67,9 +67,13 @@ pub enum Clause {
   /// The lhs must be fully processed before the rhs can be processed.
   /// Equivalent to Haskell's function of the same name
   Seq(Box<Expr>, Box<Expr>),
-  /// Insert an atom in the tree. When the clause is used in the const tree, the
-  /// atom must be trivial.
-  Atom(Atom),
+  /// Insert a new atom in the tree. When the clause is used in the const tree,
+  /// the atom must be trivial. This is always a newly constructed atom, if you
+  /// want to reference an existing atom, use the corresponding [ExprTicket].
+  /// Because the atom is newly constructed, it also must belong to this system.
+  /// For convenience, [SysId::MAX] is also accepted as referring to this
+  /// system.
+  Atom(LocalAtom),
   /// A reference to a constant
   Const(TStrv),
 }
@@ -77,7 +81,7 @@ pub enum Clause {
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub struct Expr {
   pub clause: Clause,
-  pub location: Location
+  pub location: Location,
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]

orchid-api/src/fs.rs

@@ -1,16 +0,0 @@
-use orchid_api_derive::Coding;
-use orchid_api_traits::Request;
-
-pub type FsId = u16;
-
-#[derive(Clone, Debug, Coding)]
-pub enum Loaded {
-  Code(String),
-  Collection(Vec<String>),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct FsRead(pub Vec<String>);
-impl Request for FsRead {
-  type Response = Result<Loaded, ()>;
-}

orchid-api/src/lib.rs

@@ -1,10 +1,10 @@
 pub mod atom;
 pub mod error;
 pub mod expr;
-pub mod fs;
 pub mod intern;
 pub mod location;
+pub mod parser;
 pub mod proto;
 pub mod system;
 pub mod tree;
-pub mod parser;
+pub mod vfs;

orchid-api/src/location.rs

@@ -2,11 +2,14 @@ use std::ops::Range;
 
 use orchid_api_derive::Coding;
 
-use crate::intern::TStrv;
+use crate::intern::{TStr, TStrv};
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub enum Location {
   None,
+  /// Used in functions to denote the generated code that carries on the
+  /// location of the call. Not allowed in the const tree.
+  Inherit,
   Gen(CodeGenInfo),
   Range(SourceRange),
 }
@@ -19,6 +22,6 @@ pub struct SourceRange {
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub struct CodeGenInfo {
-  pub generator: String,
-  pub details: String,
+  pub generator: TStr,
+  pub details: TStr,
 }

orchid-api/src/parser.rs

@@ -3,63 +3,50 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
+use crate::error::ProjResult;
 use crate::intern::TStr;
 use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
 use crate::tree::TokenTree;
 
+/// - All ranges contain at least one character
+/// - All ranges are in increasing characeter order
+/// - There are excluded characters between each pair of neighboring ranges
+#[derive(Clone, Debug, Coding)]
+pub struct CharFilter(pub Vec<RangeInclusive<char>>);
+
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 #[extendable]
 pub enum ParserReq {
-  MkLexer(MkLexer),
   Lex(Lex),
 }
 
-pub type LexerId = u16;
-
-#[derive(Clone, Debug, Coding)]
-pub struct Lexer {
-  id: LexerId,
-  drop: bool,
-  notify_chars: Option<Vec<RangeInclusive<char>>>,
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ParserReq, HostExtReq)]
-pub struct MkLexer(pub SysId, pub TStr);
-impl Request for MkLexer {
-  type Response = Lexer;
-}
-
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ParserReq, HostExtReq)]
 pub struct Lex {
-  pub parser: LexerId,
-  pub next: char,
+  pub sys: SysId,
+  pub text: TStr,
   pub pos: u32,
 }
 impl Request for Lex {
-  type Response = Option<LexResult>;
+  type Response = Option<ProjResult<Lexed>>;
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct LexResult {
-  pub consumed: u32,
+pub struct Lexed {
+  pub pos: u32,
   pub data: TokenTree,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct SubLex {
-  pub lexer: LexerId,
+  pub text: TStr,
   pub pos: u32,
 }
 impl Request for SubLex {
-  type Response = SubLex;
+  type Response = ProjResult<Lexed>;
 }
 
-#[derive(Clone, Debug, Coding)]
-pub struct LexerDrop;
-
 pub struct ParseLine {}

orchid-api/src/proto.rs

@@ -27,18 +27,18 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::{atom, expr, intern, parser, system, tree};
+use crate::{atom, expr, intern, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader;
 impl Decode for HostHeader {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, HOST_INTRO);
     Self
   }
 }
 impl Encode for HostHeader {
-  fn encode<W: Write>(&self, write: &mut W) { write_exact(write, HOST_INTRO) }
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) { write_exact(write, HOST_INTRO) }
 }
 
 static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
@@ -46,13 +46,13 @@ pub struct ExtensionHeader {
   pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-  fn decode<R: Read>(read: &mut R) -> Self {
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, EXT_INTRO);
     Self { systems: Vec::decode(read) }
   }
 }
 impl Encode for ExtensionHeader {
-  fn encode<W: Write>(&self, write: &mut W) {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     write_exact(write, EXT_INTRO);
     self.systems.encode(write)
   }
@@ -76,8 +76,8 @@ pub enum ExtHostReq {
 }
 
 /// Notifications sent from the extension to the host
-#[derive(Debug, Clone, Coding, Hierarchy)]
 #[allow(clippy::enum_variant_names)]
+#[derive(Debug, Clone, Coding, Hierarchy)]
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
@@ -99,6 +99,7 @@ pub enum HostExtReq {
   AtomReq(atom::AtomReq),
   ParserReq(parser::ParserReq),
   GetConstTree(tree::GetConstTree),
+  VfsReq(vfs::VfsReq),
 }
 
 /// Notifications sent from the host to the extension
@@ -107,7 +108,6 @@ pub enum HostExtReq {
 pub enum HostExtNotif {
   SystemDrop(system::SystemDrop),
   AtomDrop(atom::AtomDrop),
-  LexerDrop(parser::LexerDrop),
   /// The host can assume that after this notif is sent, a correctly written
   /// extension will eventually exit.
   Exit,

orchid-api/src/system.rs

@@ -2,6 +2,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
+use crate::parser::CharFilter;
 use crate::proto::{HostExtNotif, HostExtReq};
 
 /// ID of a system type
@@ -46,7 +47,15 @@ pub struct NewSystem {
   pub depends: Vec<SysId>,
 }
 impl Request for NewSystem {
-  type Response = ();
+  type Response = SystemInst;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct SystemInst {
+  /// The set of possible starting characters of tokens the lexer of this system
+  /// can process. The lexer will notify this system if it encounters one of
+  /// these characters.9
+  pub lex_filter: CharFilter,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/vfs.rs

@@ -0,0 +1,45 @@
+use std::collections::HashMap;
+
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::error::ProjResult;
+use crate::intern::TStr;
+use crate::proto::HostExtReq;
+use crate::system::SysId;
+
+pub type VfsId = u16;
+
+#[derive(Clone, Debug, Coding)]
+pub enum Loaded {
+  Code(String),
+  Collection(Vec<TStr>),
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(VfsReq, HostExtReq)]
+pub struct VfsRead(pub SysId, pub VfsId, pub Vec<TStr>);
+impl Request for VfsRead {
+  type Response = ProjResult<Loaded>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum EagerVfs {
+  Lazy(VfsId),
+  Eager(HashMap<TStr, EagerVfs>),
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(VfsReq, HostExtReq)]
+pub struct GetVfs(pub SysId);
+impl Request for GetVfs {
+  type Response = EagerVfs;
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+#[extendable]
+pub enum VfsReq {
+  GetVfs(GetVfs),
+  VfsRead(VfsRead),
+}