in midst of refactor

orchid-base/src/virt_fs/common.rs

@@ -0,0 +1,95 @@
+use std::rc::Rc;
+use std::sync::Arc;
+
+use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
+use crate::intern::Token;
+use crate::name::{PathSlice, VPath};
+
+/// Represents the result of loading code from a string-tree form such
+/// as the file system. Cheap to clone.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum Loaded {
+  /// Conceptually equivalent to a sourcefile
+  Code(Arc<String>),
+  /// Conceptually equivalent to the list of *.orc files in a folder, without
+  /// the extension
+  Collection(Arc<Vec<Token<String>>>),
+}
+impl Loaded {
+  /// Is the loaded item source code (not a collection)?
+  pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
+  /// Collect the elements in a collection rreport
+  pub fn collection(items: impl IntoIterator<Item = Token<String>>) -> Self {
+    Self::Collection(Arc::new(items.into_iter().collect()))
+  }
+}
+
+/// Returned by any source loading callback
+pub type FSResult = Result<Loaded, ErrorSansOriginObj>;
+
+/// Type that indicates the type of an entry without reading the contents
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub enum FSKind {
+  /// Invalid path or read error
+  None,
+  /// Source code
+  Code,
+  /// Internal tree node
+  Collection,
+}
+
+/// Distinguished error for missing code
+#[derive(Clone, PartialEq, Eq)]
+pub struct CodeNotFound(pub VPath);
+impl CodeNotFound {
+  /// Instantiate error
+  pub fn new(path: VPath) -> Self { Self(path) }
+}
+impl ErrorSansOrigin for CodeNotFound {
+  const DESCRIPTION: &'static str = "No source code for path";
+  fn message(&self) -> String { format!("{} not found", self.0) }
+}
+
+/// A simplified view of a file system for the purposes of source code loading.
+/// This includes the real FS and source code, but also various in-memory
+/// formats and other sources for libraries and dependencies.
+pub trait VirtFS {
+  /// Implementation of [VirtFS::read]
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult;
+  /// Discover information about a path without reading it.
+  ///
+  /// Implement this if your vfs backend can do expensive operations
+  fn kind(&self, path: &PathSlice) -> FSKind {
+    match self.read(path) {
+      Err(_) => FSKind::None,
+      Ok(Loaded::Code(_)) => FSKind::Code,
+      Ok(Loaded::Collection(_)) => FSKind::Collection,
+    }
+  }
+  /// Convert a path into a human-readable string that is meaningful in the
+  /// target context.
+  fn display(&self, path: &[Token<String>]) -> Option<String>;
+  /// Convert the FS handler into a type-erased version of itself for packing in
+  /// a tree.
+  fn rc(self) -> Rc<dyn VirtFS>
+  where Self: Sized + 'static {
+    Rc::new(self)
+  }
+  /// Read a path, returning either a text file, a directory listing or an
+  /// error. Wrapper for [VirtFS::get]
+  fn read(&self, path: &PathSlice) -> FSResult { self.get(path, path) }
+}
+
+impl VirtFS for &dyn VirtFS {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (*self).get(path, full_path)
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> { (*self).display(path) }
+}
+
+impl<T: VirtFS + ?Sized> VirtFS for Rc<T> {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (**self).get(path, full_path)
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> { (**self).display(path) }
+}

orchid-base/src/virt_fs/decl.rs

@@ -0,0 +1,73 @@
+use std::rc::Rc;
+use std::sync::Arc;
+
+use super::common::CodeNotFound;
+use super::{FSResult, Loaded, VirtFS};
+use crate::intern::Token;
+use crate::proj_error::ErrorSansOrigin;
+use crate::name::PathSlice;
+use crate::tree::{ModEntry, ModMember};
+use crate::combine::Combine;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ConflictingTrees;
+
+impl Combine for Rc<dyn VirtFS> {
+  type Error = ConflictingTrees;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { Err(ConflictingTrees) }
+}
+
+impl Combine for Arc<dyn VirtFS> {
+  type Error = ConflictingTrees;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { Err(ConflictingTrees) }
+}
+
+impl<'a> Combine for &'a dyn VirtFS {
+  type Error = ConflictingTrees;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { Err(ConflictingTrees) }
+}
+
+/// A declarative in-memory tree with [VirtFS] objects for leaves. Paths are
+/// followed to a leaf and the leftover handled by it.
+pub type DeclTree = ModEntry<Rc<dyn VirtFS>, (), ()>;
+
+impl VirtFS for DeclTree {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    match &self.member {
+      ModMember::Item(it) => it.get(path, full_path),
+      ModMember::Sub(module) => match path.split_first() {
+        None => Ok(Loaded::collection(module.keys(|_| true))),
+        Some((head, tail)) => (module.entries.get(head))
+          .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
+          .and_then(|ent| ent.get(tail, full_path)),
+      },
+    }
+  }
+
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    let (head, tail) = path.split_first()?;
+    match &self.member {
+      ModMember::Item(it) => it.display(path),
+      ModMember::Sub(module) => module.entries.get(head)?.display(tail),
+    }
+  }
+}
+
+impl VirtFS for String {
+  fn display(&self, _: &[Token<String>]) -> Option<String> { None }
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (path.is_empty().then(|| Loaded::Code(Arc::new(self.as_str().to_string()))))
+      .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
+  }
+}
+
+impl<'a> VirtFS for &'a str {
+  fn display(&self, _: &[Token<String>]) -> Option<String> { None }
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (path.is_empty().then(|| Loaded::Code(Arc::new(self.to_string()))))
+      .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
+  }
+}
+
+/// Insert a file by cleartext contents in the [DeclTree].
+pub fn decl_file(s: &str) -> DeclTree { DeclTree::leaf(Rc::new(s.to_string())) }

orchid-base/src/virt_fs/dir.rs

@@ -0,0 +1,121 @@
+use std::cell::RefCell;
+use std::fs::File;
+use std::io;
+use std::io::{ErrorKind, Read};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex};
+
+use hashbrown::HashMap;
+
+use super::common::CodeNotFound;
+use super::{FSResult, Loaded, VirtFS};
+use crate::intern::{intern, Token};
+use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
+use crate::name::PathSlice;
+
+#[derive(Clone)]
+struct OpenError {
+  file: Arc<Mutex<io::Error>>,
+  dir: Arc<Mutex<io::Error>>,
+}
+impl OpenError {
+  pub fn wrap(file: io::Error, dir: io::Error) -> ErrorSansOriginObj {
+    Self { dir: Arc::new(Mutex::new(dir)), file: Arc::new(Mutex::new(file)) }.pack()
+  }
+}
+impl ErrorSansOrigin for OpenError {
+  const DESCRIPTION: &'static str = "A file system error occurred";
+  fn message(&self) -> String {
+    let Self { dir, file } = self;
+    format!(
+      "File system errors other than not found occurred\n\
+      as a file: {}\nas a directory: {}",
+      file.lock().unwrap(),
+      dir.lock().unwrap()
+    )
+  }
+}
+
+#[derive(Clone)]
+struct IOError(Arc<Mutex<io::Error>>);
+impl IOError {
+  pub fn wrap(inner: io::Error) -> ErrorSansOriginObj { Self(Arc::new(Mutex::new(inner))).pack() }
+}
+impl ErrorSansOrigin for IOError {
+  const DESCRIPTION: &'static str = "an I/O error occured";
+  fn message(&self) -> String { format!("File read error: {}", self.0.lock().unwrap()) }
+}
+
+#[derive(Clone)]
+struct NotUtf8(PathBuf);
+impl NotUtf8 {
+  pub fn wrap(path: &Path) -> ErrorSansOriginObj { Self(path.to_owned()).pack() }
+}
+impl ErrorSansOrigin for NotUtf8 {
+  const DESCRIPTION: &'static str = "Source files must be UTF-8";
+  fn message(&self) -> String {
+    format!("{} is a source file but contains invalid UTF-8", self.0.display())
+  }
+}
+
+/// A real file system directory linked into the virtual FS
+pub struct DirNode {
+  cached: RefCell<HashMap<PathBuf, FSResult>>,
+  root: PathBuf,
+  suffix: &'static str,
+}
+impl DirNode {
+  /// Reference a real file system directory in the virtual FS
+  pub fn new(root: PathBuf, suffix: &'static str) -> Self {
+    assert!(suffix.starts_with('.'), "Extension must begin with .");
+    Self { cached: RefCell::default(), root, suffix }
+  }
+
+  fn ext(&self) -> &str { self.suffix.strip_prefix('.').expect("Checked in constructor") }
+
+  fn load_file(&self, fpath: &Path, orig_path: &PathSlice) -> FSResult {
+    match fpath.read_dir() {
+      Err(dir_e) => {
+        let fpath = fpath.with_extension(self.ext());
+        let mut file =
+          File::open(&fpath).map_err(|file_e| match (dir_e.kind(), file_e.kind()) {
+            (ErrorKind::NotFound, ErrorKind::NotFound) =>
+              CodeNotFound::new(orig_path.to_vpath()).pack(),
+            _ => OpenError::wrap(file_e, dir_e),
+          })?;
+        let mut buf = Vec::new();
+        file.read_to_end(&mut buf).map_err(IOError::wrap)?;
+        let text = String::from_utf8(buf).map_err(|_| NotUtf8::wrap(&fpath))?;
+        Ok(Loaded::Code(Arc::new(text)))
+      },
+      Ok(dir) => Ok(Loaded::collection(dir.filter_map(|ent_r| {
+        let ent = ent_r.ok()?;
+        let name = ent.file_name().into_string().ok()?;
+        match ent.metadata().ok()?.is_dir() {
+          false => Some(intern(name.strip_suffix(self.suffix)?)),
+          true => Some(intern(&name)),
+        }
+      }))),
+    }
+  }
+
+  fn mk_pathbuf(&self, path: &[Token<String>]) -> PathBuf {
+    let mut fpath = self.root.clone();
+    path.iter().for_each(|seg| fpath.push(seg.as_str()));
+    fpath
+  }
+}
+impl VirtFS for DirNode {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    let fpath = self.mk_pathbuf(path);
+    let mut binding = self.cached.borrow_mut();
+    let (_, res) = (binding.raw_entry_mut().from_key(&fpath))
+      .or_insert_with(|| (fpath.clone(), self.load_file(&fpath, full_path)));
+    res.clone()
+  }
+
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    let pathbuf = self.mk_pathbuf(path).with_extension(self.ext());
+    Some(pathbuf.to_string_lossy().to_string())
+  }
+}

orchid-base/src/virt_fs/embed.rs

@@ -0,0 +1,74 @@
+use std::sync::Arc;
+
+use itertools::Itertools as _;
+use rust_embed::RustEmbed;
+
+use super::common::CodeNotFound;
+use super::{FSResult, Loaded, VirtFS};
+use crate::intern::{intern, Token};
+use crate::proj_error::ErrorSansOrigin;
+use crate::location::CodeGenInfo;
+use crate::name::PathSlice;
+use crate::tree::{ModEntry, ModMember, Module};
+
+/// An in-memory FS tree for libraries managed internally by the interpreter
+pub struct EmbeddedFS {
+  tree: Module<Arc<String>, (), ()>,
+  suffix: &'static str,
+  gen: CodeGenInfo,
+}
+impl EmbeddedFS {
+  /// Expose a directory embedded in a binary wiht [RustEmbed] to the
+  /// interpreter
+  pub fn new<T: RustEmbed>(suffix: &'static str, gen: CodeGenInfo) -> Self {
+    let mut tree = Module::wrap([]);
+    for path in T::iter() {
+      let data_buf = T::get(&path).expect("path from iterator").data.to_vec();
+      let data = String::from_utf8(data_buf).expect("embed must be utf8");
+      let mut cur_node = &mut tree;
+      let path_no_suffix = path.strip_suffix(suffix).expect("embed filtered for suffix");
+      let mut segments = path_no_suffix.split('/').map(intern);
+      let mut cur_seg = segments.next().expect("Embed is a directory");
+      for next_seg in segments {
+        if !cur_node.entries.contains_key(&cur_seg) {
+          let ent = ModEntry::wrap(ModMember::Sub(Module::wrap([])));
+          cur_node.entries.insert(cur_seg.clone(), ent);
+        }
+        let ent = cur_node.entries.get_mut(&cur_seg).expect("just constructed");
+        match &mut ent.member {
+          ModMember::Sub(submod) => cur_node = submod,
+          _ => panic!("Aliased file and folder"),
+        };
+        cur_seg = next_seg;
+      }
+      let data_ent = ModEntry::wrap(ModMember::Item(Arc::new(data)));
+      let prev = cur_node.entries.insert(cur_seg, data_ent);
+      debug_assert!(prev.is_none(), "file name unique");
+    }
+    // if gen.generator == "std" {
+    //   panic!(
+    //     "{:?}",
+    //     tree.map_data(&|_, s| (), &|_, x| x, &|_, x| x, Substack::Bottom)
+    //   );
+    // };
+    Self { gen, suffix, tree }
+  }
+}
+
+impl VirtFS for EmbeddedFS {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    if path.is_empty() {
+      return Ok(Loaded::collection(self.tree.keys(|_| true)));
+    }
+    let entry = (self.tree.walk1_ref(&[], path, |_| true))
+      .map_err(|_| CodeNotFound::new(full_path.to_vpath()).pack())?;
+    Ok(match &entry.0.member {
+      ModMember::Item(text) => Loaded::Code(text.clone()),
+      ModMember::Sub(sub) => Loaded::collection(sub.keys(|_| true)),
+    })
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    let Self { gen, suffix, .. } = self;
+    Some(format!("{}{suffix} in {gen}", path.iter().join("/")))
+  }
+}

orchid-base/src/virt_fs/mod.rs

@@ -0,0 +1,18 @@
+//! Abstractions and primitives to help define the namespace tree used by
+//! Orchid.
+//!
+//! Although this may make it seem like the namespace tree is very flexible,
+//! libraries are generally permitted and expected to hardcode their own
+//! location in the tree, so it's up to the embedder to ensure that the flexible
+//! structure retains the assumed location of all code.
+mod common;
+mod decl;
+mod dir;
+mod embed;
+mod prefix;
+
+pub use common::{CodeNotFound, FSKind, FSResult, Loaded, VirtFS};
+pub use decl::{decl_file, DeclTree};
+pub use dir::DirNode;
+pub use embed::EmbeddedFS;
+pub use prefix::PrefixFS;

orchid-base/src/virt_fs/prefix.rs

@@ -0,0 +1,38 @@
+use itertools::Itertools;
+
+use super::common::CodeNotFound;
+use super::VirtFS;
+use crate::intern::Token;
+use crate::name::{PathSlice, VPath};
+use crate::proj_error::ErrorSansOrigin;
+
+/// Modify the prefix of a nested file tree
+pub struct PrefixFS<'a> {
+  remove: VPath,
+  add: VPath,
+  wrapped: Box<dyn VirtFS + 'a>,
+}
+impl<'a> PrefixFS<'a> {
+  /// Modify the prefix of a file tree
+  pub fn new(wrapped: impl VirtFS + 'a, remove: impl AsRef<str>, add: impl AsRef<str>) -> Self {
+    Self {
+      wrapped: Box::new(wrapped),
+      remove: VPath::parse(remove.as_ref()),
+      add: VPath::parse(add.as_ref()),
+    }
+  }
+  fn proc_path(&self, path: &[Token<String>]) -> Option<Vec<Token<String>>> {
+    let path = path.strip_prefix(self.remove.as_slice())?;
+    Some(self.add.0.iter().chain(path).cloned().collect_vec())
+  }
+}
+impl<'a> VirtFS for PrefixFS<'a> {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> super::FSResult {
+    let path =
+      self.proc_path(path).ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())?;
+    self.wrapped.get(&path, full_path)
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    self.wrapped.display(&self.proc_path(path)?)
+  }
+}