Gitbutler >:(

I don't understand this piece of software at all

orchid-base/src/error.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 use itertools::Itertools;
 
 use crate::api;
-use crate::interner::{deintern, Tok};
+use crate::interner::Tok;
 use crate::location::Pos;
 
 /// A point of interest in resolving the error, such as the point where
@@ -18,21 +18,21 @@ pub struct ErrPos {
   pub message: Option<Arc<String>>,
 }
 impl ErrPos {
-  pub fn from_api(pel: &api::ErrLocation) -> Self {
+  pub fn new(msg: &str, position: Pos) -> Self {
+    Self { message: Some(Arc::new(msg.to_string())), position }
+  }
+  fn from_api(api: &api::ErrLocation) -> Self {
     Self {
-      message: Some(pel.message.clone()).filter(|s| !s.is_empty()),
-      position: Pos::from_api(&pel.location),
+      message: Some(api.message.clone()).filter(|s| !s.is_empty()),
+      position: Pos::from_api(&api.location),
     }
   }
-  pub fn to_api(&self) -> api::ErrLocation {
+  fn to_api(&self) -> api::ErrLocation {
     api::ErrLocation {
       message: self.message.clone().unwrap_or_default(),
       location: self.position.to_api(),
     }
   }
-  pub fn new(msg: &str, position: Pos) -> Self {
-    Self { message: Some(Arc::new(msg.to_string())), position }
-  }
 }
 impl From<Pos> for ErrPos {
   fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
@@ -45,20 +45,20 @@ pub struct OrcErr {
   pub positions: Vec<ErrPos>,
 }
 impl OrcErr {
-  pub fn from_api(err: &api::OrcError) -> Self {
-    Self {
-      description: deintern(err.description),
-      message: err.message.clone(),
-      positions: err.locations.iter().map(ErrPos::from_api).collect(),
-    }
-  }
-  pub fn to_api(&self) -> api::OrcError {
+  fn to_api(&self) -> api::OrcError {
     api::OrcError {
-      description: self.description.marker(),
+      description: self.description.to_api(),
       message: self.message.clone(),
       locations: self.positions.iter().map(ErrPos::to_api).collect(),
     }
   }
+  fn from_api(api: &api::OrcError) -> Self {
+    Self {
+      description: Tok::from_api(api.description),
+      message: api.message.clone(),
+      positions: api.locations.iter().map(ErrPos::from_api).collect(),
+    }
+  }
 }
 impl Eq for OrcErr {}
 impl PartialEq for OrcErr {
@@ -90,14 +90,6 @@ impl OrcErrv {
     if v.is_empty() { Err(EmptyErrv) } else { Ok(Self(v)) }
   }
   #[must_use]
-  pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect_vec() }
-  #[must_use]
-  pub fn from_api<'a>(apiv: impl IntoIterator<Item = &'a api::OrcError>) -> Self {
-    let v = apiv.into_iter().map(OrcErr::from_api).collect_vec();
-    assert!(!v.is_empty(), "Error condition with 0 errors");
-    Self(v)
-  }
-  #[must_use]
   pub fn extended<T>(mut self, errors: impl IntoIterator<Item = T>) -> Self
   where Self: Extend<T> {
     self.extend(errors);
@@ -119,6 +111,10 @@ impl OrcErrv {
   pub fn pos_iter(&self) -> impl Iterator<Item = ErrPos> + '_ {
     self.0.iter().flat_map(|e| e.positions.iter().cloned())
   }
+  pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect() }
+  pub fn from_api<'a>(api: impl IntoIterator<Item = &'a api::OrcError>) -> Self {
+    Self(api.into_iter().map(OrcErr::from_api).collect())
+  }
 }
 impl From<OrcErr> for OrcErrv {
   fn from(value: OrcErr) -> Self { Self(vec![value]) }

orchid-base/src/interner.rs

@@ -10,7 +10,6 @@ use itertools::Itertools as _;
 use orchid_api_traits::{Decode, Encode, Request};
 
 use crate::api;
-use orchid_api_traits::{ApiEquiv, FromApi, ToApi};
 use crate::reqnot::{DynRequester, Requester};
 
 /// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
@@ -25,7 +24,10 @@ pub struct Tok<T: Interned> {
 }
 impl<T: Interned> Tok<T> {
   pub fn new(data: Arc<T>, marker: T::Marker) -> Self { Self { data, marker: ForceSized(marker) } }
-  pub fn marker(&self) -> T::Marker { self.marker.0 }
+  pub fn to_api(&self) -> T::Marker { self.marker.0 }
+  pub fn from_api<M>(marker: M) -> Self where M: InternMarker<Interned = T> {
+    deintern(marker)
+  }
   pub fn arc(&self) -> Arc<T> { self.data.clone() }
 }
 impl<T: Interned> Deref for Tok<T> {
@@ -34,7 +36,7 @@ impl<T: Interned> Deref for Tok<T> {
   fn deref(&self) -> &Self::Target { self.data.as_ref() }
 }
 impl<T: Interned> Ord for Tok<T> {
-  fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.marker().cmp(&other.marker()) }
+  fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.to_api().cmp(&other.to_api()) }
 }
 impl<T: Interned> PartialOrd for Tok<T> {
   fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
@@ -44,7 +46,7 @@ impl<T: Interned> PartialEq for Tok<T> {
   fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
 }
 impl<T: Interned> hash::Hash for Tok<T> {
-  fn hash<H: hash::Hasher>(&self, state: &mut H) { self.marker().hash(state) }
+  fn hash<H: hash::Hasher>(&self, state: &mut H) { self.to_api().hash(state) }
 }
 impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -53,7 +55,7 @@ impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
 }
 impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-    write!(f, "Token({} -> {:?})", self.marker().get_id(), self.data.as_ref())
+    write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
   }
 }
 impl<T: Interned + Encode> Encode for Tok<T> {
@@ -117,25 +119,13 @@ impl Internable for String {
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_string()) }
 }
 
-impl ApiEquiv for Tok<String> {
-  type Api = api::TStr;
-}
-impl ToApi for Tok<String> {
-  type Ctx = ();
-  fn to_api(&self, _: &mut Self::Ctx) -> Self::Api { self.marker() }
-}
-impl FromApi for Tok<String> {
-  type Ctx = ();
-  fn from_api(api: &Self::Api, _: &mut Self::Ctx) -> Self { deintern(*api) }
-}
-
 impl Interned for Vec<Tok<String>> {
   type Marker = api::TStrv;
   fn intern(
     self: Arc<Self>,
     req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
   ) -> Self::Marker {
-    req.request(api::InternStrv(Arc::new(self.iter().map(|t| t.marker()).collect())))
+    req.request(api::InternStrv(Arc::new(self.iter().map(|t| t.to_api()).collect())))
   }
   fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
 }
@@ -172,17 +162,6 @@ impl Internable for [api::TStr] {
     Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
   }
 }
-impl ApiEquiv for Tok<Vec<Tok<String>>> {
-  type Api = api::TStrv;
-}
-impl ToApi for Tok<Vec<Tok<String>>> {
-  type Ctx = ();
-  fn to_api(&self, _: &mut Self::Ctx) -> Self::Api { self.marker() }
-}
-impl FromApi for Tok<Vec<Tok<String>>> {
-  type Ctx = ();
-  fn from_api(api: &Self::Api, _: &mut Self::Ctx) -> Self { deintern(*api) }
-}
 
 /// The number of references held to any token by the interner.
 const BASE_RC: usize = 3;
@@ -202,7 +181,7 @@ pub struct Bimap<T: Interned> {
 impl<T: Interned> Bimap<T> {
   pub fn insert(&mut self, token: Tok<T>) {
     self.intern.insert(token.data.clone(), token.clone());
-    self.by_id.insert(token.marker(), token);
+    self.by_id.insert(token.to_api(), token);
   }
 
   pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> { self.by_id.get(&marker).cloned() }
@@ -218,14 +197,14 @@ impl<T: Interned> Bimap<T> {
     (self.intern)
       .extract_if(|k, _| Arc::strong_count(k) == BASE_RC)
       .map(|(_, v)| {
-        self.by_id.remove(&v.marker());
-        v.marker()
+        self.by_id.remove(&v.to_api());
+        v.to_api()
       })
       .collect()
   }
 
   pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
-    self.intern.retain(|k, v| BASE_RC < Arc::strong_count(k) || retained.contains(&v.marker()))
+    self.intern.retain(|k, v| BASE_RC < Arc::strong_count(k) || retained.contains(&v.to_api()))
   }
 }
 
@@ -298,7 +277,7 @@ pub fn intern<T: Interned>(t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<
   tok
 }
 
-pub fn deintern<M: InternMarker>(marker: M) -> Tok<M::Interned> {
+fn deintern<M: InternMarker>(marker: M) -> Tok<M::Interned> {
   let mut g = interner();
   if let Some(tok) = M::Interned::bimap(&mut g.interners).by_marker(marker) {
     return tok;

orchid-base/src/join.rs

@@ -12,7 +12,11 @@ pub fn join_maps<K: Eq + Hash, V>(
   right: HashMap<K, V>,
   mut merge: impl FnMut(&K, V, V) -> V,
 ) -> HashMap<K, V> {
-  try_join_maps(left, right, |k, l, r| Ok(merge(k, l, r))).unwrap_or_else(|e: Never| match e {})
+  let (val, ev) = try_join_maps::<K, V, Never>(left, right, |k, l, r| Ok(merge(k, l, r)));
+  if let Some(e) = ev.first() {
+    match *e {}
+  }
+  val
 }
 
 /// Combine two hashmaps via a fallible value merger. See also [join_maps]
@@ -20,15 +24,22 @@ pub fn try_join_maps<K: Eq + Hash, V, E>(
   left: HashMap<K, V>,
   mut right: HashMap<K, V>,
   mut merge: impl FnMut(&K, V, V) -> Result<V, E>,
-) -> Result<HashMap<K, V>, E> {
+) -> (HashMap<K, V>, Vec<E>) {
   let mut mixed = HashMap::with_capacity(left.len() + right.len());
+  let mut errors = Vec::new();
   for (key, lval) in left {
     let val = match right.remove(&key) {
       None => lval,
-      Some(rval) => merge(&key, lval, rval)?,
+      Some(rval) => match merge(&key, lval, rval) {
+        Ok(v) => v,
+        Err(e) => {
+          errors.push(e);
+          continue;
+        },
+      },
     };
     mixed.insert(key, val);
   }
   mixed.extend(right);
-  Ok(mixed)
+  (mixed, errors)
 }

orchid-base/src/lib.rs

@@ -19,6 +19,7 @@ pub mod parse;
 pub mod pure_seq;
 pub mod reqnot;
 pub mod sequence;
+pub mod side;
 pub mod tokens;
 pub mod tree;
 pub mod macros;

orchid-base/src/location.rs

@@ -7,8 +7,7 @@ use std::ops::Range;
 
 use trait_set::trait_set;
 
-use orchid_api_traits::{ApiEquiv, FromApi, ToApi};
-use crate::interner::{deintern, intern, Tok};
+use crate::interner::{intern, Tok};
 use crate::name::Sym;
 use crate::{api, intern, sym};
 
@@ -38,30 +37,20 @@ impl Pos {
       other => format!("{other:?}"),
     }
   }
-}
-impl ApiEquiv for Pos {
-  type Api = api::Location;
-}
-impl FromApi for Pos {
-  type Ctx = ();
-  fn from_api(api: &Self::Api, ctx: &mut Self::Ctx) -> Self {
+  pub fn from_api(api: &api::Location) -> Self {
     match_mapping!(api, api::Location => Pos {
       None, Inherit, SlotTarget,
       Range(r.clone()),
-      Gen(cgi => CodeGenInfo::from_api(cgi, &mut ())),
-      SourceRange(sr => CodeGenInfo::from_api(sr, &mut ()))
+      Gen(cgi => CodeGenInfo::from_api(cgi)),
+      SourceRange(sr => SourceRange::from_api(sr))
     })
   }
-}
-
-impl ToApi for Pos {
-  type Ctx = ();
-  fn to_api(&self, ctx: &mut Self::Ctx) -> Self::Api {
-    match_mapping!(self, Pos => Self::Api {
+  pub fn to_api(&self) -> api::Location {
+    match_mapping!(self, Pos => api::Location {
       None, Inherit, SlotTarget,
       Range(r.clone()),
-      Gen(cgi.to_api(ctx)),
-      SourceRange(sr.to_api(ctx)),
+      Gen(cgi.to_api()),
+      SourceRange(sr.to_api()),
     })
   }
 }
@@ -106,20 +95,11 @@ impl SourceRange {
   pub fn zw(path: Sym, pos: u32) -> Self {
     Self { path, range: pos..pos }
   }
-}
-impl ApiEquiv for SourceRange {
-  type Api = api::SourceRange;
-}
-impl FromApi for SourceRange {
-  type Ctx = ();
-  fn from_api(api: &Self::Api, ctx: &mut Self::Ctx) -> Self {
-    Self { path: Sym::from_api(&api.path, ctx), range: api.range.clone() }
+  fn from_api(api: &api::SourceRange) -> Self {
+    Self { path: Sym::from_api(api.path), range: api.range.clone() }
   }
-}
-impl ToApi for SourceRange {
-  type Ctx = ();
-  fn to_api(&self, ctx: &mut Self::Ctx) -> Self::Api {
-    api::SourceRange { path: self.path.to_api(ctx), range: self.range.clone() }
+  fn to_api(&self) -> api::SourceRange {
+    api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
   }
 }
 
@@ -140,6 +120,15 @@ impl CodeGenInfo {
   }
   /// Syntactic location
   pub fn pos(&self) -> Pos { Pos::Gen(self.clone()) }
+  fn from_api(api: &api::CodeGenInfo) -> Self {
+    Self {
+      generator: Sym::from_api(api.generator),
+      details: Tok::from_api(api.details),
+    }
+  }
+  fn to_api(&self) -> api::CodeGenInfo {
+    api::CodeGenInfo { generator: self.generator.to_api(), details: self.details.to_api() }
+  }
 }
 impl fmt::Debug for CodeGenInfo {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeGenInfo({self})") }
@@ -150,24 +139,6 @@ impl fmt::Display for CodeGenInfo {
     if !self.details.is_empty() { write!(f, ", details: {}", self.details) } else { write!(f, ".") }
   }
 }
-impl ApiEquiv for CodeGenInfo {
-  type Api = api::CodeGenInfo;
-}
-impl FromApi for CodeGenInfo {
-  type Ctx = ();
-  fn from_api(api: &Self::Api, ctx: &mut Self::Ctx) -> Self {
-    Self {
-      generator: Sym::from_api(&api.generator, ctx),
-      details: Tok::from_api(&api.details, ctx),
-    }
-  }
-}
-impl ToApi for CodeGenInfo {
-  type Ctx = ();
-  fn to_api(&self, ctx: &mut Self::Ctx) -> Self::Api {
-    api::CodeGenInfo { generator: self.generator.to_api(ctx), details: self.details.to_api(ctx) }
-  }
-}
 
 #[must_use]
 fn pos2lc(s: &str, i: u32) -> (u32, u32) {

orchid-base/src/macros.rs

@@ -1,17 +1,35 @@
 use itertools::Itertools;
+use never::Never;
+use trait_set::trait_set;
 
-use crate::{name::Sym, tree::{AtomTok, Paren, Ph, TokTree}};
-use std::marker::PhantomData;
+use crate::{match_mapping, name::Sym, tree::{Paren, Ph}};
+use std::{marker::PhantomData, sync::Arc};
 
 use crate::{api, location::Pos};
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct MacroSlot<'a>(api::MacroTreeId, PhantomData<&'a ()>);
+impl<'a> MacroSlot<'a> {
+  pub fn id(self) -> api::MacroTreeId { self.0 }
+}
+
+trait_set! {
+  pub trait MacroAtomToApi<A> = FnMut(&A) -> api::MacroToken;
+  pub trait MacroAtomFromApi<'a, A> = FnMut(&api::Atom) -> MTok<'a, A>;
+}
 
 #[derive(Clone, Debug)]
-pub struct MTree<'a, A: AtomTok> {
+pub struct MTree<'a, A> {
   pub pos: Pos,
-  pub tok: MTok<'a, A>
+  pub tok: Arc<MTok<'a, A>>
+}
+impl<'a, A> MTree<'a, A> {
+  pub(crate) fn from_api(api: &api::MacroTree, do_atom: &mut impl MacroAtomFromApi<'a, A>) -> Self {
+    Self { pos: Pos::from_api(&api.location), tok: Arc::new(MTok::from_api(&api.token, do_atom)) }
+  }
+  pub(crate) fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroTree {
+    api::MacroTree { location: self.pos.to_api(), token: self.tok.to_api(do_atom) }
+  }
 }
 
 #[derive(Clone, Debug)]
@@ -21,41 +39,48 @@ pub enum MTok<'a, A> {
   Slot(MacroSlot<'a>),
   Lambda(Vec<MTree<'a, A>>, Vec<MTree<'a, A>>),
   Ph(Ph),
-  Atom(A)
+  Atom(A),
+  Ref(Box<MTok<'a, Never>>),
 }
-impl<'a, A> MTree<'a, A> {
-  pub(crate) fn from_api(api: &api::MacroTree) -> Self {
-    use api::MacroToken as MTK;
-    let tok = match &api.token {
-      MTK::Lambda(x, b) => MTok::Lambda(mtreev_from_api(x), mtreev_from_api(b)),
-      MTK::Name(t) => MTok::Name(Sym::deintern(*t)),
-      MTK::Slot(tk) => MTok::Slot(MacroSlot(tk.clone(), PhantomData)),
-      MTK::S(p, b) => MTok::S(p.clone(), mtreev_from_api(b)),
-      MTK::Ph(ph) => MTok::Ph(Ph::from_api(ph)),
-    };
-    Self { pos: Pos::from_api(&api.location), tok }
+impl<'a, A> MTok<'a, A> {
+  pub(crate) fn from_api(
+    api: &api::MacroToken,
+    do_atom: &mut impl MacroAtomFromApi<'a, A>
+  ) -> Self {
+    match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
+      Lambda(x => mtreev_from_api(x, do_atom), b => mtreev_from_api(b, do_atom)),
+      Name(t => Sym::from_api(*t)),
+      Slot(tk => MacroSlot(*tk, PhantomData)),
+      S(p.clone(), b => mtreev_from_api(b, do_atom)),
+      Ph(ph => Ph::from_api(ph)),
+    } {
+      api::MacroToken::Atom(a) => do_atom(a)
+    })
   }
-  pub(crate) fn to_api(&self) -> api::MacroTree {
-    use api::MacroToken as MTK;
-    let token = match &self.tok {
-      MTok::Lambda(x, b) => MTK::Lambda(mtreev_to_api(x), mtreev_to_api(b)),
-      MTok::Name(t) => MTK::Name(t.tok().marker()),
-      MTok::Ph(ph) => MTK::Ph(ph.to_api()),
-      MTok::S(p, b) => MTK::S(p.clone(), mtreev_to_api(b)),
-      MTok::Slot(tk) => MTK::Slot(tk.0.clone()),
-    };
-    api::MacroTree { location: self.pos.to_api(), token }
+  pub(crate) fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroToken {
+    match_mapping!(&self, MTok => api::MacroToken {
+      Lambda(x => mtreev_to_api(x, do_atom), b => mtreev_to_api(b, do_atom)),
+      Name(t.tok().to_api()),
+      Ph(ph.to_api()),
+      S(p.clone(), b => mtreev_to_api(b, do_atom)),
+      Slot(tk.0.clone()),
+    } {
+      MTok::Ref(r) => r.to_api(&mut |e| match *e {}),
+      MTok::Atom(a) => do_atom(a),
+    })
   }
 }
 
 pub fn mtreev_from_api<'a, 'b, A>(
-  api: impl IntoIterator<Item = &'b api::MacroTree>
+  api: impl IntoIterator<Item = &'b api::MacroTree>,
+  do_atom: &mut impl MacroAtomFromApi<'a, A>
 ) -> Vec<MTree<'a, A>> {
-  api.into_iter().map(MTree::from_api).collect_vec()
+  api.into_iter().map(|api| MTree::from_api(api, do_atom)).collect_vec()
 }
 
 pub fn mtreev_to_api<'a: 'b, 'b, A: 'b>(
-  v: impl IntoIterator<Item = &'b MTree<'a, A>>
+  v: impl IntoIterator<Item = &'b MTree<'a, A>>,
+  do_atom: &mut impl MacroAtomToApi<A>
 ) -> Vec<api::MacroTree> {
-  v.into_iter().map(MTree::to_api).collect_vec()
+  v.into_iter().map(|t| t.to_api(do_atom)).collect_vec()
 }

orchid-base/src/match_mapping.rs

@@ -12,84 +12,108 @@
 /// ```
 #[macro_export]
 macro_rules! match_mapping {
-  ($input:expr, $src:ty => $tgt:ty {
+  // Entry point
+  ($input:expr, $($src:ident)::* => $tgt:ty {
     $($branches:tt)*
-  }) => {
-    match_mapping!(@BRANCH_MUNCH (($input) ($src) ($tgt)) () $($branches)* ,)
+  } $({
+    $($extra:tt)*
+  })?) => {
+    match_mapping!(@BRANCH_MUNCH
+      (($input) ($($src)*) ($tgt) ($($($extra)*)?))
+      ()
+      $($branches)* ,
+    )
+    // note: we're adding a comma to the input so the optional trailing comma becomes
+    // an optional second comma which is easier to match
   };
-  (@BRANCHES_DONE ( ($input:expr) ($src:ty) ($tgt:ty) )
+  // ======== Process match branches
+  // Can't generate branches individually so gather them into a collection and render them here
+  (@BRANCHES_DONE ( ($input:expr) $src:tt ($tgt:ty) ($($extra:tt)*) )
     $( ( $variant:ident $($pat:tt)*) )*
   ) => {
     {
-      use $src as Foo;
       match $input {
         $(
-          match_mapping!(@PAT (Foo :: $variant) $($pat)*) =>
+          match_mapping!(@PAT ($src $variant) $($pat)*) =>
             match_mapping!(@VAL (< $tgt >:: $variant) $($pat)*),
         )*
+        $($extra)*
       }
     }
   };
+  // End with optional second comma
   (@BRANCH_MUNCH $ext:tt ( $($branches:tt)* ) $(,)?) => {
     match_mapping!(@BRANCHES_DONE $ext $($branches)* )
   };
+  // Unit variant
   (@BRANCH_MUNCH $ext:tt ( $($branches:tt)* ) $variant:ident , $($tail:tt)*) => {
     match_mapping!(@BRANCH_MUNCH $ext ( $($branches)* ($variant) ) $($tail)*)
   };
+  // Variant mapped to same shape pair
   (@BRANCH_MUNCH $ext:tt ( $($branches:tt)* ) $variant:ident $pat:tt , $($tail:tt)*) => {
     match_mapping!(@BRANCH_MUNCH $ext
       ( $($branches)* ($variant $pat) )
     $($tail)*)
   };
-  (@PAT ($($prefix:tt)*) ( $($fields:tt)* )) => {
-    $($prefix)* ( match_mapping!(@PAT_MUNCH () $($fields)*) )
+  (@PAT (($($prefix:tt)*) $variant:ident)) => { $($prefix ::)* $variant };
+  (@PAT $prefix:tt ( $($fields:tt)* )) => {
+    match_mapping!(@PAT_MUNCH (() $prefix) () $($fields)* ,)
   };
-  (@PAT ($($prefix:tt)*) { $($fields:tt)* }) => {
-    $($prefix)* { match_mapping!(@PAT_MUNCH () $($fields)*) }
+  (@PAT $prefix:tt { $($fields:tt)* }) => {
+    match_mapping!(@PAT_MUNCH ({} $prefix) () $($fields)* ,)
   };
-  (@PAT ($($path:tt)*)) => { $($path)* };
-  (@PAT_MUNCH ($($names:ident)*) $name:ident => $value:expr) => { $($names ,)* $name };
-  (@PAT_MUNCH ($($names:ident)*) $name:ident => $value:expr , $($tail:tt)*) => {
-    match_mapping!(@PAT_MUNCH ($($names)* $name) $($tail)*)
+  (@PAT_MUNCH (() (($($prefix:ident)*) $variant:ident)) ($($names:ident)*)) => {
+    $($prefix)::* :: $variant ( $($names),* )
   };
-  (@PAT_MUNCH ($($names:ident)*) $name:ident . $($tail:tt)*) => { 
-    match_mapping!(@PAT_DOT_MUNCH ($($names)* $name) $($tail)*)
+  (@PAT_MUNCH ({} (($($prefix:ident)*) $variant:ident)) ($($names:ident)*)) => {
+    $($prefix)::* :: $variant { $($names),* }
   };
-  (@PAT_MUNCH ($($names:ident)*)) => { $($names),* };
-  (@PAT_DOT_MUNCH $names:tt , $($tail:tt)*) => {
-    match_mapping!(@PAT_MUNCH $names $($tail)*)
+  (@PAT_MUNCH $ctx:tt $names:tt $(,)? ) => { match_mapping!($ctx $names) };
+  (@PAT_MUNCH $ctx:tt ($($names:ident)*) * $name:ident , $($tail:tt)*) => {
+    match_mapping!(@PAT_MUNCH $ctx ($($names)* $name) $($tail)*)
   };
-  (@PAT_DOT_MUNCH $names:tt $_:tt $($tail:tt)*) => {
-    match_mapping!(@PAT_DOT_MUNCH $names $($tail)*)
+  (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident => $value:expr , $($tail:tt)*) => {
+    match_mapping!(@PAT_MUNCH $ctx ($($names)* $name) $($tail)*)
+  };
+  (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident . $($tail:tt)*) => { 
+    match_mapping!(@PAT_DOT_MUNCH $ctx ($($names)* $name) $($tail)*)
+  };
+  (@PAT_DOT_MUNCH $ctx:tt $names:tt , $($tail:tt)*) => {
+    match_mapping!(@PAT_MUNCH $ctx $names $($tail)*)
+  };
+  (@PAT_DOT_MUNCH $ctx:tt $names:tt $_:tt $($tail:tt)*) => {
+    match_mapping!(@PAT_DOT_MUNCH $ctx $names $($tail)*)
   };
-  (@PAT_DOT_MUNCH ($($names:tt)*)) => { $($names),* };
   (@VAL ($($prefix:tt)*)) => { $($prefix)* };
-  (@VAL ($($prefix:tt)*) ( $($fields:tt)* )) => {
-    $($prefix)* ( match_mapping!(@VAL_MUNCH () () $($fields)* ) )
+  (@VAL $prefix:tt ( $($fields:tt)* )) => {
+    match_mapping!(@VAL_MUNCH (() $prefix) () $($fields)* , )
   };
-  (@VAL ($($prefix:tt)*) { $($fields:tt)* }) => {
-    $($prefix)* { match_mapping!(@VAL_MUNCH {} () $($fields)* ) }
+  (@VAL $prefix:tt { $($fields:tt)* }) => {
+    match_mapping!(@VAL_MUNCH ({} $prefix) () $($fields)* , )
   };
-  (@VAL_MUNCH () ($($prefix:tt)*) $name:ident => $value:expr) => { $($prefix)* $value };
-  (@VAL_MUNCH () ($($prefix:tt)*) $name:ident => $value:expr , $($tail:tt)*) => {
-    match_mapping!(@VAL_MUNCH () ($($prefix)* $value, ) $($tail)*)
+  (@VAL_MUNCH $ctx:tt ($($prefix:tt)*) * $name:ident , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH $ctx ($($prefix)* ($name (* $name)) ) $($tail)*)
   };
-  (@VAL_MUNCH {} ($($prefix:tt)*) $name:ident => $value:expr) => { $($prefix)* $name: $value };
-  (@VAL_MUNCH {} ($($prefix:tt)*) $name:ident => $value:expr , $($tail:tt)*) => {
-    match_mapping!(@VAL_MUNCH {} ($($prefix)* $name: $value, ) $($tail)*)
+  (@VAL_MUNCH $ctx:tt ($($prefix:tt)*) $name:ident => $value:expr , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH $ctx ($($prefix)* ($name ($value)) ) $($tail)*)
   };
-  (@VAL_MUNCH () ($($prefix:tt)*) $name:ident . $member:tt $($tail:tt)*) => {
-    match_mapping!(@VAL_DOT_MUNCH () ($($prefix)* $name . $member ) $($tail)*)
+  (@VAL_MUNCH $ctx:tt $fields:tt $name:ident . $member:tt $($tail:tt)*) => {
+    match_mapping!(@VAL_DOT_MUNCH $ctx $fields $name ($name . $member ) $($tail)*)
   };
-  (@VAL_MUNCH {} ($($prefix:tt)*) $name:ident . $member:tt $($tail:tt)*) => {
-    match_mapping!(@VAL_DOT_MUNCH {} ($($prefix)* $name: $name . $member) $($tail)*)
+  (@VAL_DOT_MUNCH $ctx:tt ($($fields:tt)*) $name:ident $current:tt , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH $ctx ($($fields)* ($name $current)) $($tail)*)
   };
-  (@VAL_DOT_MUNCH $ptyp:tt ($($prefix:tt)*) , $($tail:tt)*) => {
-    match_mapping!(@VAL_MUNCH $ptyp ($($prefix)* ,) $($tail)*)
+  (@VAL_DOT_MUNCH $ctx:tt $fields:tt $name:ident ($($current:tt)*) $tt:tt $($tail:tt)*) => {
+    match_mapping!(@VAL_DOT_MUNCH $ctx $fields $name ($($current)* $tt) $($tail)*)
   };
-  (@VAL_DOT_MUNCH $ptyp:tt ($($prefix:tt)*) $tt:tt $($tail:tt)*) => {
-    match_mapping!(@VAL_DOT_MUNCH $ptyp ($($prefix)* $tt) $($tail)*)
+  (@VAL_DOT_MUNCH $ctx:tt ($($fields:tt)*) $name:ident $current:tt) => {
+    match_mapping!(@VAL_MUNCH $ptyp ($($fields)* ($name $current)))
+  };
+  (@VAL_MUNCH $ctx:tt $fields:tt , ) => { match_mapping!(@VAL_MUNCH $ctx $fields) };
+  (@VAL_MUNCH (() ($($prefix:tt)*)) ($( ( $name:ident $($value:tt)* ) )*) ) => {
+    $($prefix)* ( $( $($value)* ),* )
+  };
+  (@VAL_MUNCH ({} ($($prefix:tt)*)) ($( ( $name:ident $($value:tt)* ) )*) ) => {
+    $($prefix)* { $( $name : $($value)* ),* }
   };
-  (@VAL_DOT_MUNCH $ptyp:tt ($($prefix:tt)*)) => { $($prefix)* };
-  (@VAL_MUNCH $_ptyp:tt ($($prefix:tt)*)) => { $($prefix)* };
 }

orchid-base/src/name.rs

@@ -12,8 +12,7 @@ use itertools::Itertools;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::api_conv::{ApiEquiv, FromApi, ToApi};
-use crate::interner::{deintern, intern, InternMarker, Tok};
+use crate::interner::{intern, InternMarker, Tok};
 
 trait_set! {
   /// Traits that all name iterators should implement
@@ -258,7 +257,7 @@ impl VName {
     if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
   }
   pub fn deintern(items: impl IntoIterator<Item = api::TStr>) -> Result<Self, EmptyNameError> {
-    Self::new(items.into_iter().map(deintern))
+    Self::new(items.into_iter().map(Tok::from_api))
   }
   /// Unwrap the enclosed vector
   pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
@@ -354,12 +353,13 @@ impl Sym {
   /// Grab the interner token
   pub fn tok(&self) -> Tok<Vec<Tok<String>>> { self.0.clone() }
   /// Get a number unique to this name suitable for arbitrary ordering.
-  pub fn id(&self) -> NonZeroU64 { self.0.marker().get_id() }
+  pub fn id(&self) -> NonZeroU64 { self.0.to_api().get_id() }
   /// Extern the sym for editing
   pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
-  pub fn deintern(marker: api::TStrv) -> Sym {
-    Self::from_tok(deintern(marker)).expect("Empty sequence found for serialized Sym")
+  pub fn from_api(marker: api::TStrv) -> Sym {
+    Self::from_tok(Tok::from_api(marker)).expect("Empty sequence found for serialized Sym")
   }
+  pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
 }
 impl fmt::Debug for Sym {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Sym({self})") }
@@ -386,17 +386,6 @@ impl Deref for Sym {
   type Target = PathSlice;
   fn deref(&self) -> &Self::Target { self.borrow() }
 }
-impl ApiEquiv for Sym {
-  type Api = api::TStrv;
-}
-impl<C> ToApi<C> for Sym {
-  fn to_api(&self, ctx: &mut C) -> Self::Api { self.tok().to_api(ctx) }
-}
-impl<C> FromApi<C> for Sym {
-  fn from_api(api: &Self::Api, ctx: &mut C) -> Self {
-    Self::from_tok(Tok::from_api(api, ctx)).expect("Empty sequence found for serialized Sym")
-  }
-}
 
 /// An abstraction over tokenized vs non-tokenized names so that they can be
 /// handled together in datastructures. The names can never be empty

orchid-base/src/parse.rs

@@ -4,10 +4,10 @@ use std::ops::{Deref, Range};
 use itertools::Itertools;
 
 use crate::error::{mk_err, mk_errv, OrcRes, Reporter};
-use crate::interner::{deintern, intern, Tok};
+use crate::interner::{intern, Tok};
 use crate::location::Pos;
 use crate::name::VPath;
-use crate::tree::{AtomTok, ExtraTok, Paren, TokTree, Token};
+use crate::tree::{AtomRepr, ExtraTok, Paren, TokTree, Token};
 use crate::{api, intern};
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
@@ -16,11 +16,11 @@ pub fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{
 pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 
 #[derive(Debug)]
-pub struct Snippet<'a, 'b, A: AtomTok, X: ExtraTok> {
+pub struct Snippet<'a, 'b, A: AtomRepr, X: ExtraTok> {
   prev: &'a TokTree<'b, A, X>,
   cur: &'a [TokTree<'b, A, X>],
 }
-impl<'a, 'b, A: AtomTok, X: ExtraTok> Snippet<'a, 'b, A, X> {
+impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
   pub fn new(prev: &'a TokTree<'b, A, X>, cur: &'a [TokTree<'b, A, X>]) -> Self {
     Self { prev, cur }
   }
@@ -67,18 +67,18 @@ impl<'a, 'b, A: AtomTok, X: ExtraTok> Snippet<'a, 'b, A, X> {
     self.split_at(non_fluff_start.unwrap_or(self.len())).1
   }
 }
-impl<'a, 'b, A: AtomTok, X: ExtraTok> Copy for Snippet<'a, 'b, A, X> {}
-impl<'a, 'b, A: AtomTok, X: ExtraTok> Clone for Snippet<'a, 'b, A, X> {
+impl<'a, 'b, A: AtomRepr, X: ExtraTok> Copy for Snippet<'a, 'b, A, X> {}
+impl<'a, 'b, A: AtomRepr, X: ExtraTok> Clone for Snippet<'a, 'b, A, X> {
   fn clone(&self) -> Self { *self }
 }
-impl<'a, 'b, A: AtomTok, X: ExtraTok> Deref for Snippet<'a, 'b, A, X> {
+impl<'a, 'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'a, 'b, A, X> {
   type Target = [TokTree<'b, A, X>];
   fn deref(&self) -> &Self::Target { self.cur }
 }
 
 /// Remove tokens that aren't meaningful in expression context, such as comments
 /// or line breaks
-pub fn strip_fluff<'a, A: AtomTok, X: ExtraTok>(
+pub fn strip_fluff<'a, A: AtomRepr, X: ExtraTok>(
   tt: &TokTree<'a, A, X>,
 ) -> Option<TokTree<'a, A, X>> {
   let tok = match &tt.tok {
@@ -97,15 +97,15 @@ pub struct Comment {
   pub pos: Pos,
 }
 impl Comment {
-  pub fn from_api(api: &api::Comment) -> Self {
-    Self { pos: Pos::from_api(&api.location), text: deintern(api.text) }
-  }
   pub fn to_api(&self) -> api::Comment {
-    api::Comment { location: self.pos.to_api(), text: self.text.marker() }
+    api::Comment { location: self.pos.to_api(), text: self.text.to_api() }
+  }
+  pub fn from_api(api: &api::Comment) -> Self {
+    Self { pos: Pos::from_api(&api.location), text: Tok::from_api(api.text) }  
   }
 }
 
-pub fn line_items<'a, 'b, A: AtomTok, X: ExtraTok>(
+pub fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
   snip: Snippet<'a, 'b, A, X>,
 ) -> Vec<Parsed<'a, 'b, Vec<Comment>, A, X>> {
   let mut items = Vec::new();
@@ -131,7 +131,7 @@ pub fn line_items<'a, 'b, A: AtomTok, X: ExtraTok>(
   items
 }
 
-pub fn try_pop_no_fluff<'a, 'b, A: AtomTok, X: ExtraTok>(
+pub fn try_pop_no_fluff<'a, 'b, A: AtomRepr, X: ExtraTok>(
   snip: Snippet<'a, 'b, A, X>,
 ) -> ParseRes<'a, 'b, &'a TokTree<'b, A, X>, A, X> {
   snip.skip_fluff().pop_front().map(|(output, tail)| Parsed { output, tail }).ok_or_else(|| {
@@ -143,7 +143,7 @@ pub fn try_pop_no_fluff<'a, 'b, A: AtomTok, X: ExtraTok>(
   })
 }
 
-pub fn expect_end(snip: Snippet<'_, '_, impl AtomTok, impl ExtraTok>) -> OrcRes<()> {
+pub fn expect_end(snip: Snippet<'_, '_, impl AtomRepr, impl ExtraTok>) -> OrcRes<()> {
   match snip.skip_fluff().get(0) {
     Some(surplus) => Err(mk_errv(
       intern!(str: "Extra code after end of line"),
@@ -154,7 +154,7 @@ pub fn expect_end(snip: Snippet<'_, '_, impl AtomTok, impl ExtraTok>) -> OrcRes<
   }
 }
 
-pub fn expect_tok<'a, 'b, A: AtomTok, X: ExtraTok>(
+pub fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>(
   snip: Snippet<'a, 'b, A, X>,
   tok: Tok<String>,
 ) -> ParseRes<'a, 'b, (), A, X> {
@@ -169,20 +169,20 @@ pub fn expect_tok<'a, 'b, A: AtomTok, X: ExtraTok>(
   }
 }
 
-pub struct Parsed<'a, 'b, T, A: AtomTok, X: ExtraTok> {
+pub struct Parsed<'a, 'b, T, A: AtomRepr, X: ExtraTok> {
   pub output: T,
   pub tail: Snippet<'a, 'b, A, X>,
 }
 
 pub type ParseRes<'a, 'b, T, A, X> = OrcRes<Parsed<'a, 'b, T, A, X>>;
 
-pub fn parse_multiname<'a, 'b, A: AtomTok, X: ExtraTok>(
+pub fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
   ctx: &impl Reporter,
   tail: Snippet<'a, 'b, A, X>,
 ) -> ParseRes<'a, 'b, Vec<(Import, Pos)>, A, X> {
   let ret = rec(ctx, tail);
   #[allow(clippy::type_complexity)] // it's an internal function
-  pub fn rec<'a, 'b, A: AtomTok, X: ExtraTok>(
+  pub fn rec<'a, 'b, A: AtomRepr, X: ExtraTok>(
     ctx: &impl Reporter,
     tail: Snippet<'a, 'b, A, X>,
   ) -> ParseRes<'a, 'b, Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, A, X> {

orchid-base/src/side.rs

@@ -0,0 +1,97 @@
+//! Named left/right. I tried bools, I couldn't consistently remember which one
+//! is left, so I made an enum. Rust should optimize this into a bool anyway.
+
+use std::fmt;
+use std::ops::Not;
+
+use crate::boxed_iter::BoxedIter;
+
+/// A primitive for encoding the two sides Left and Right. While booleans
+/// are technically usable for this purpose, they're very easy to confuse
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Side {
+  /// Left, low, or high-to-low in the case of sequences
+  Left,
+  /// Right, high, or low-to-high in the case of sequences
+  Right,
+}
+
+impl fmt::Display for Side {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Left => write!(f, "Left"),
+      Self::Right => write!(f, "Right"),
+    }
+  }
+}
+
+impl Side {
+  /// Get the side that is not the current one
+  pub fn opposite(&self) -> Self {
+    match self {
+      Self::Left => Self::Right,
+      Self::Right => Self::Left,
+    }
+  }
+  /// Shorthand for opposite
+  pub fn inv(&self) -> Self { self.opposite() }
+  /// take N elements from this end of a slice
+  pub fn slice<'a, T>(&self, size: usize, slice: &'a [T]) -> &'a [T] {
+    match self {
+      Side::Left => &slice[..size],
+      Side::Right => &slice[slice.len() - size..],
+    }
+  }
+  /// ignore N elements from this end of a slice
+  pub fn crop<'a, T>(&self, margin: usize, slice: &'a [T]) -> &'a [T] {
+    self.opposite().slice(slice.len() - margin, slice)
+  }
+  /// ignore N elements from this end and M elements from the other end
+  /// of a slice
+  pub fn crop_both<'a, T>(&self, margin: usize, opposite: usize, slice: &'a [T]) -> &'a [T] {
+    self.crop(margin, self.opposite().crop(opposite, slice))
+  }
+  /// Pick this side from a pair of things
+  pub fn pick<T>(&self, pair: (T, T)) -> T {
+    match self {
+      Side::Left => pair.0,
+      Side::Right => pair.1,
+    }
+  }
+  /// Make a pair with the first element on this side
+  pub fn pair<T>(&self, this: T, opposite: T) -> (T, T) {
+    match self {
+      Side::Left => (this, opposite),
+      Side::Right => (opposite, this),
+    }
+  }
+  /// Walk a double ended iterator (assumed to be left-to-right) in this
+  /// direction
+  pub fn walk<'a, I: DoubleEndedIterator + 'a>(&self, iter: I) -> BoxedIter<'a, I::Item> {
+    match self {
+      Side::Right => Box::new(iter) as BoxedIter<I::Item>,
+      Side::Left => Box::new(iter.rev()),
+    }
+  }
+}
+
+impl Not for Side {
+  type Output = Side;
+
+  fn not(self) -> Self::Output { self.opposite() }
+}
+
+#[cfg(test)]
+mod test {
+  use itertools::Itertools;
+
+  use super::*;
+
+  /// I apparently have a tendency to mix these up so it's best if
+  /// the sides are explicitly stated
+  #[test]
+  fn test_walk() {
+    assert_eq!(Side::Right.walk(0..4).collect_vec(), vec![0, 1, 2, 3], "can walk a range");
+    assert_eq!(Side::Left.walk(0..4).collect_vec(), vec![3, 2, 1, 0], "can walk a range backwards")
+  }
+}

orchid-base/src/tree.rs

@@ -8,13 +8,13 @@ use std::sync::Arc;
 
 use itertools::Itertools;
 use never::Never;
-use orchid_api::Placeholder;
 use ordered_float::NotNan;
 use trait_set::trait_set;
 
-use crate::api;
+use crate::{api, match_mapping};
 use crate::error::OrcErrv;
-use crate::interner::{deintern, Tok};
+use crate::interner::Tok;
+use crate::location::Pos;
 use crate::name::PathSlice;
 use crate::parse::Snippet;
 use crate::tokens::PARENS;
@@ -22,11 +22,11 @@ use crate::tokens::PARENS;
 pub use api::PhKind as PhKind;
 
 trait_set! {
-  pub trait RecurCB<'a, A: AtomTok, X: ExtraTok> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
+  pub trait RecurCB<'a, A: AtomRepr, X: ExtraTok> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
   pub trait ExtraTok = Display + Clone + fmt::Debug;
 }
 
-pub fn recur<'a, A: AtomTok, X: ExtraTok>(
+pub fn recur<'a, A: AtomRepr, X: ExtraTok>(
   tt: TokTree<'a, A, X>,
   f: &impl Fn(TokTree<'a, A, X>, &dyn RecurCB<'a, A, X>) -> TokTree<'a, A, X>,
 ) -> TokTree<'a, A, X> {
@@ -42,14 +42,14 @@ pub fn recur<'a, A: AtomTok, X: ExtraTok>(
   })
 }
 
-pub trait AtomTok: fmt::Display + Clone + fmt::Debug {
-  type Context: ?Sized;
-  fn from_api(atom: &api::Atom, pos: Range<u32>, ctx: &mut Self::Context) -> Self;
-  fn to_api(&self) -> api::Atom;
+pub trait AtomRepr: fmt::Display + Clone + fmt::Debug {
+  type Ctx: ?Sized;
+  fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self;
+  fn to_api(&self) -> orchid_api::Atom;
 }
-impl AtomTok for Never {
-  type Context = Never;
-  fn from_api(_: &api::Atom, _: Range<u32>, _: &mut Self::Context) -> Self { panic!() }
+impl AtomRepr for Never {
+  type Ctx = Never;
+  fn from_api(_: &api::Atom, _: Pos, _: &mut Self::Ctx) -> Self { panic!() }
   fn to_api(&self) -> orchid_api::Atom { match *self {} }
 }
 
@@ -66,25 +66,24 @@ impl<'a> Display for TokHandle<'a> {
 }
 
 #[derive(Clone, Debug)]
-pub struct TokTree<'a, A: AtomTok, X: ExtraTok> {
+pub struct TokTree<'a, A: AtomRepr, X: ExtraTok> {
   pub tok: Token<'a, A, X>,
   pub range: Range<u32>,
 }
-impl<'a, A: AtomTok, X: ExtraTok> TokTree<'a, A, X> {
-  pub fn from_api(tt: &api::TokenTree, ctx: &mut A::Context) -> Self {
-    let tok = match &tt.token {
-      api::Token::Atom(a) => Token::Atom(A::from_api(a, tt.range.clone(), ctx)),
-      api::Token::BR => Token::BR,
-      api::Token::NS => Token::NS,
-      api::Token::Bottom(e) => Token::Bottom(OrcErrv::from_api(e)),
-      api::Token::LambdaHead(arg) => Token::LambdaHead(ttv_from_api(arg, ctx)),
-      api::Token::Name(name) => Token::Name(deintern(*name)),
-      api::Token::S(par, b) => Token::S(*par, ttv_from_api(b, ctx)),
-      api::Token::Comment(c) => Token::Comment(c.clone()),
-      api::Token::Slot(id) => Token::Slot(TokHandle::new(*id)),
-      api::Token::Ph(ph) => Token::Ph(Ph {name: deintern(ph.name), kind: ph.kind }),
-      api::Token::Macro(prio) => Token::Macro(*prio)
-    };
+impl<'a, A: AtomRepr, X: ExtraTok> TokTree<'a, A, X> {
+  pub fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx) -> Self {
+    let tok = match_mapping!(&tt.token, api::Token => Token::<'a, A, X> {
+      BR, NS,
+      Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx)),
+      Bottom(e => OrcErrv::from_api(e)),
+      LambdaHead(arg => ttv_from_api(arg, ctx)),
+      Name(n => Tok::from_api(*n)),
+      S(*par, b => ttv_from_api(b, ctx)),
+      Comment(c.clone()),
+      Slot(id => TokHandle::new(*id)),
+      Ph(ph => Ph::from_api(ph)),
+      Macro(*prio)
+    });
     Self { range: tt.range.clone(), tok }
   }
 
@@ -92,20 +91,21 @@ impl<'a, A: AtomTok, X: ExtraTok> TokTree<'a, A, X> {
     &self,
     do_extra: &mut impl FnMut(&X, Range<u32>) -> api::TokenTree,
   ) -> api::TokenTree {
-    let token = match &self.tok {
-      Token::Atom(a) => api::Token::Atom(a.to_api()),
-      Token::BR => api::Token::BR,
-      Token::NS => api::Token::NS,
-      Token::Bottom(e) => api::Token::Bottom(e.to_api()),
-      Token::Comment(c) => api::Token::Comment(c.clone()),
-      Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_to_api(arg, do_extra)),
-      Token::Name(n) => api::Token::Name(n.marker()),
-      Token::Slot(tt) => api::Token::Slot(tt.ticket()),
-      Token::S(p, b) => api::Token::S(*p, ttv_to_api(b, do_extra)),
-      Token::Ph(Ph { name, kind }) => api::Token::Ph(Placeholder { name: name.marker(), kind: *kind }),
-      Token::X(x) => return do_extra(x, self.range.clone()),
-      Token::Macro(prio) => api::Token::Macro(*prio),
-    };
+    let token = match_mapping!(&self.tok, Token => api::Token {
+      Atom(a.to_api()),
+      BR,
+      NS,
+      Bottom(e.to_api()),
+      Comment(c.clone()),
+      LambdaHead(arg => ttv_to_api(arg, do_extra)),
+      Name(n.to_api()),
+      Slot(tt.ticket()),
+      S(*p, b => ttv_to_api(b, do_extra)),
+      Ph(ph.to_api()),
+      Macro(*prio),
+    } {
+      Token::X(x) => return do_extra(x, self.range.clone())
+    });
     api::TokenTree { range: self.range.clone(), token }
   }
 
@@ -120,10 +120,11 @@ impl<'a, A: AtomTok, X: ExtraTok> TokTree<'a, A, X> {
       Token::Bottom(e) => api::Token::Bottom(e.to_api()),
       Token::Comment(c) => api::Token::Comment(c.clone()),
       Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra)),
-      Token::Name(n) => api::Token::Name(n.marker()),
+      Token::Name(n) => api::Token::Name(n.to_api()),
       Token::Slot(tt) => api::Token::Slot(tt.ticket()),
       Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra)),
-      Token::Ph(Ph { kind, name }) => api::Token::Ph(Placeholder { name: name.marker(), kind }),
+      Token::Ph(Ph { kind, name }) =>
+        api::Token::Ph(api::Placeholder { name: name.to_api(), kind }),
       Token::X(x) => return do_extra(x, self.range.clone()),
       Token::Macro(prio) => api::Token::Macro(prio),
     };
@@ -145,25 +146,25 @@ impl<'a, A: AtomTok, X: ExtraTok> TokTree<'a, A, X> {
   }
 }
 
-impl<'a, A: AtomTok, X: ExtraTok> Display for TokTree<'a, A, X> {
+impl<'a, A: AtomRepr, X: ExtraTok> Display for TokTree<'a, A, X> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
 }
 
-pub fn ttv_from_api<A: AtomTok, X: ExtraTok>(
+pub fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
   tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
-  ctx: &mut A::Context,
+  ctx: &mut A::Ctx,
 ) -> Vec<TokTree<'static, A, X>> {
   tokv.into_iter().map(|t| TokTree::<A, X>::from_api(t.borrow(), ctx)).collect()
 }
 
-pub fn ttv_to_api<'a, A: AtomTok, X: ExtraTok>(
+pub fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>(
   tokv: impl IntoIterator<Item: Borrow<TokTree<'a, A, X>>>,
   do_extra: &mut impl FnMut(&X, Range<u32>) -> api::TokenTree,
 ) -> Vec<api::TokenTree> {
   tokv.into_iter().map(|tok| Borrow::<TokTree<A, X>>::borrow(&tok).to_api(do_extra)).collect_vec()
 }
 
-pub fn ttv_into_api<'a, A: AtomTok, X: ExtraTok>(
+pub fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
   tokv: impl IntoIterator<Item = TokTree<'a, A, X>>,
   do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
 ) -> Vec<api::TokenTree> {
@@ -172,7 +173,7 @@ pub fn ttv_into_api<'a, A: AtomTok, X: ExtraTok>(
 
 /// This takes a position and not a range because it assigns the range to
 /// multiple leaf tokens, which is only valid if it's a zero-width range
-pub fn vname_tv<'a: 'b, 'b, A: AtomTok + 'a, X: ExtraTok + 'a>(
+pub fn vname_tv<'a: 'b, 'b, A: AtomRepr + 'a, X: ExtraTok + 'a>(
   name: &'b PathSlice,
   pos: u32,
 ) -> impl Iterator<Item = TokTree<'a, A, X>> + 'b {
@@ -182,7 +183,7 @@ pub fn vname_tv<'a: 'b, 'b, A: AtomTok + 'a, X: ExtraTok + 'a>(
     .map(move |t| t.at(pos..pos))
 }
 
-pub fn wrap_tokv<'a, A: AtomTok, X: ExtraTok>(
+pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
   items: impl IntoIterator<Item = TokTree<'a, A, X>>
 ) -> TokTree<'a, A, X> {
   let items_v = items.into_iter().collect_vec();
@@ -199,7 +200,7 @@ pub fn wrap_tokv<'a, A: AtomTok, X: ExtraTok>(
 pub use api::Paren;
 
 #[derive(Clone, Debug)]
-pub enum Token<'a, A: AtomTok, X: ExtraTok> {
+pub enum Token<'a, A: AtomRepr, X: ExtraTok> {
   Comment(Arc<String>),
   LambdaHead(Vec<TokTree<'a, A, X>>),
   Name(Tok<String>),
@@ -213,7 +214,7 @@ pub enum Token<'a, A: AtomTok, X: ExtraTok> {
   Ph(Ph),
   Macro(Option<NotNan<f64>>),
 }
-impl<'a, A: AtomTok, X: ExtraTok> Token<'a, A, X> {
+impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
   pub fn at(self, range: Range<u32>) -> TokTree<'a, A, X> { TokTree { range, tok: self } }
   pub fn is_kw(&self, tk: Tok<String>) -> bool {
     matches!(self, Token::Name(n) if *n == tk)
@@ -225,7 +226,7 @@ impl<'a, A: AtomTok, X: ExtraTok> Token<'a, A, X> {
     }
   }
 }
-impl<'a, A: AtomTok, X: ExtraTok> Display for Token<'a, A, X> {
+impl<'a, A: AtomRepr, X: ExtraTok> Display for Token<'a, A, X> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     thread_local! {
       static PAREN_LEVEL: RefCell<usize> = 0.into();
@@ -270,13 +271,13 @@ impl<'a, A: AtomTok, X: ExtraTok> Display for Token<'a, A, X> {
   }
 }
 
-pub fn ttv_range(ttv: &[TokTree<'_, impl AtomTok, impl ExtraTok>]) -> Range<u32> {
+pub fn ttv_range(ttv: &[TokTree<'_, impl AtomRepr, impl ExtraTok>]) -> Range<u32> {
   assert!(!ttv.is_empty(), "Empty slice has no range");
   ttv.first().unwrap().range.start..ttv.last().unwrap().range.end
 }
 
 pub fn ttv_fmt<'a: 'b, 'b>(
-  ttv: impl IntoIterator<Item = &'b TokTree<'a, impl AtomTok + 'b, impl ExtraTok + 'b>>,
+  ttv: impl IntoIterator<Item = &'b TokTree<'a, impl AtomRepr + 'b, impl ExtraTok + 'b>>,
 ) -> String {
   ttv.into_iter().join("")
 }
@@ -297,8 +298,12 @@ pub struct Ph {
   pub kind: PhKind,
 }
 impl Ph {
-  pub fn from_api(api: &Placeholder) -> Self { Self { name: deintern(api.name), kind: api.kind } }
-  pub fn to_api(&self) -> Placeholder { Placeholder { name: self.name.marker(), kind: self.kind } }
+  pub fn from_api(api: &api::Placeholder) -> Self {
+    Self { name: Tok::from_api(api.name), kind: api.kind }
+  }
+  pub fn to_api(&self) -> api::Placeholder {
+    api::Placeholder { name: self.name.to_api(), kind: self.kind }
+  }
 }
 
 #[cfg(test)]