Initial networking module.

.cargo/config.toml

@@ -1,12 +1,11 @@
 [alias]
 xtask = "run --quiet --package xtask --"
-orcx = "xtask orcx --"
+orcx = "xtask orcx"
-orcxdb = "xtask orcxdb --"
+orcxdb = "xtask orcxdb"
 
 [env]
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
-ORCHID_EXTENSIONS = "target/debug/orchid_std"
+ORCHID_EXTENSIONS = "target/debug/orchid-std"
-#ORCHID_EXTENSIONS = "target/debug/orchid-std-piped"
 ORCHID_DEFAULT_SYSTEMS = "orchid::std;orchid::macros"
 ORCHID_LOG_BUFFERS = "true"
 RUST_BACKTRACE = "1"

.github/FUNDING.yml

@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: lbfalvy
+patreon: lbfalvy
+open_collective: # Replace with a single Open Collective username
+ko_fi: lbfalvy
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/rust.yml

@@ -2,15 +2,17 @@ name: Rust
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "master" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "master" ]
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
   build:
+    if: ${{ false }} # <- This make sure the workflow is skipped without any alert
+
     runs-on: ubuntu-latest
 
     steps:
@@ -22,8 +24,6 @@ jobs:
     - name: Run tests
       run: $HOME/.cargo/bin/cargo test --verbose
     - name: Clippy
-      run: $HOME/.cargo/bin/cargo clippy
+      run: cargo clippy
     - name: Formatting
-      run: $HOME/.cargo/bin/cargo +nightly fmt --check
+      run: cargo fmt +nightly --check
-    - name: No unqualified imports from orchid_api
-      run: $HOME/.cargo/bin/cargo xtask check-api-refs

Cargo.toml

@@ -11,7 +11,5 @@ members = [
   "orchid-api-derive",
   "orchid-api-traits",
   "stdio-perftest",
-	"xtask",
+  "xtask", "async-fn-stream",
-	"async-fn-stream",
-	"unsync-pipe",
 ]

LICENCE

@@ -1,11 +1,9 @@
-THIS SOFTWARE IS PROVIDED WITHOUT WARRANTY
-
 The code in this repository is free for noncommercial use, including derivative works and inclusion in other software if those are also free for noncommercial use. Commercial use, or inclusion in any derivative works licensed for commercial use is forbidden under this general licence.
 
 Identifying marks stored in the repository are restricted for use with an unmodified copy of this software. If you distribute modified versions of this software, you must either replace these identifying marks or modify them in a way that clearly indicates that what you are distributing is a derivative work and not this official vversion. You must also replace any contact information in such a way that your derivative work does not suggest that we may be contacted about issues. Your derivative work may use the original identifying marks and contact information to identify this project as its basis, while emphasizing that the authors of the original project are neither in control of, nor liable for the derivative work.
 
-Identifying marks include the Orchid logo, the ribbon image in the readme, and the names "Orchid", "Orchidlang" unless they are part of a technical interface.
+Identifying marks include the Orchid logo, the ribbon image above, and the names "Orchid", "Orchidlang" unless they are part of a technical interface.
 
 Contact information includes email addresses, links to the source code and issue tracker.
 
-Words listed as identifying marks are explicltly not considered as such when they appear in technical interfaces or APIs. For example, shell commands, identifiers within Orchid or Rust code, and names in package registries are not considered identifying marks.
+Words listed as identifying marks are explicltly not considered as such when they appear in technical interfaces or APIs. For example, shell commands, identifiers within Orchid or Rust code, and names in package registries are not considered as identifying marks.

NETWORKING.md

@@ -0,0 +1,283 @@
+# Networking in Orchid
+
+This document explains how to use TCP networking in the Orchid programming language.
+
+## Prerequisites
+
+Build the project with the nightly Rust toolchain:
+
+```sh
+rustup run nightly cargo build
+```
+
+## Socket Module Overview
+
+The socket module is available at `std::socket::tcp` and provides functions for TCP client/server communication.
+
+### Importing
+
+```orchid
+import std::socket::tcp::(bind, accept, connect, read, write_all, close)
+```
+
+Or import all functions:
+
+```orchid
+import std::socket::tcp
+```
+
+## API Reference
+
+### Server Functions
+
+| Function | Signature | Description |
+|----------|-----------|-------------|
+| `bind` | `String -> TcpListener` | Create a TCP listener bound to an address |
+| `accept` | `TcpListener -> TcpStream` | Accept an incoming connection |
+| `listener_addr` | `TcpListener -> String` | Get the address the listener is bound to |
+
+### Client Functions
+
+| Function | Signature | Description |
+|----------|-----------|-------------|
+| `connect` | `String -> TcpStream` | Connect to a TCP server |
+
+### Stream Functions
+
+| Function | Signature | Description |
+|----------|-----------|-------------|
+| `read` | `TcpStream -> Int -> String` | Read up to N bytes as UTF-8 string |
+| `read_bytes` | `TcpStream -> Int -> String` | Read up to N bytes as raw byte string |
+| `read_exact` | `TcpStream -> Int -> String` | Read exactly N bytes |
+| `write` | `TcpStream -> String -> Int` | Write data, returns bytes written |
+| `write_all` | `TcpStream -> String -> Int` | Write all data |
+| `flush` | `TcpStream -> Int` | Flush the stream buffer |
+| `close` | `TcpStream -> Int` | Close the connection |
+| `peer_addr` | `TcpStream -> String` | Get the remote peer's address |
+| `local_addr` | `TcpStream -> String` | Get the local address |
+
+## Quick Start Examples
+
+### Testing Socket Bind
+
+```sh
+rustup run nightly cargo orcx -- exec "std::socket::tcp::bind \"127.0.0.1:8080\""
+```
+
+Output:
+```
+<TcpListener 127.0.0.1:8080>
+```
+
+### Testing Connection (expect error if no server running)
+
+```sh
+rustup run nightly cargo orcx -- exec "std::socket::tcp::connect \"127.0.0.1:8080\""
+```
+
+Output (if no server):
+```
+error: IO error: Connection refused (os error 111):  @
+```
+
+## Example Scripts
+
+### TCP Echo Server
+
+Location: `examples/tcp-echo-server/src/main.orc`
+
+```orchid
+import std::socket::tcp::(bind, accept, read, write_all, close)
+
+const handle_client := \client. do cps {
+  cps data = read client 1024;
+  cps _ = write_all client data;
+  cps _ = close client;
+  cps pass 0;
+}
+
+const accept_loop := \server. do cps {
+  cps client = accept server;
+  cps _ = handle_client client;
+  cps pass $ accept_loop server;
+}
+
+const main := do cps {
+  cps server = bind "127.0.0.1:8080";
+  cps pass $ accept_loop server;
+}
+```
+
+Run the echo server:
+
+```sh
+rustup run nightly cargo orcx -- exec --proj ./examples/tcp-echo-server "src::main::main"
+```
+
+### TCP Client
+
+Location: `examples/tcp-client/src/main.orc`
+
+```orchid
+import std::socket::tcp::(connect, read, write_all, close, peer_addr)
+import system::io::println
+
+const main := do cps {
+  cps conn = connect "127.0.0.1:8080";
+  cps addr = peer_addr conn;
+  cps println $ "Connected to ${addr}";
+  cps _ = write_all conn "Hello from Orchid!\n";
+  cps response = read conn 1024;
+  cps println $ "Server response: ${response}";
+  cps _ = close conn;
+  cps pass 0;
+}
+```
+
+Run the client (requires a server running on port 8080):
+
+```sh
+rustup run nightly cargo orcx -- exec --proj ./examples/tcp-client "src::main::main"
+```
+
+### HTTP Server
+
+Location: `examples/http-server/src/main.orc`
+
+```orchid
+import std::socket::tcp::(bind, accept, read, write_all, close, peer_addr, listener_addr)
+import system::io::println
+
+const http_response := "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 44\r\nConnection: close\r\n\r\n<html><body><h1>Hello Orchid!</h1></body></html>"
+
+const handle_client := \client. do cps {
+  cps addr = peer_addr client;
+  cps println $ "Client connected: ${addr}";
+  cps request = read client 4096;
+  cps println $ "Received request from ${addr}";
+  cps _ = write_all client http_response;
+  cps _ = close client;
+  cps println $ "Connection closed: ${addr}";
+  cps pass 0;
+}
+
+const accept_loop := \server. do cps {
+  cps client = accept server;
+  cps _ = handle_client client;
+  cps pass $ accept_loop server;
+}
+
+const main := do cps {
+  cps server = bind "127.0.0.1:8080";
+  cps addr = listener_addr server;
+  cps println $ "HTTP Server listening on ${addr}";
+  cps println "Press Ctrl+C to stop";
+  cps pass $ accept_loop server;
+}
+```
+
+Run the HTTP server:
+
+```sh
+rustup run nightly cargo orcx -- exec --proj ./examples/http-server "src::main::main"
+```
+
+Then open http://127.0.0.1:8080 in your browser or test with curl:
+
+```sh
+curl http://127.0.0.1:8080
+```
+
+## Understanding the CPS Syntax
+
+Orchid uses Continuation-Passing Style (CPS) for side effects. The `do cps { ... }` block is used for sequential operations:
+
+- `cps variable = expression;` - Bind the result of an expression to a variable
+- `cps expression;` - Execute an expression for its side effects
+- `cps pass value;` - Return a value from the block
+
+Example breakdown:
+
+```orchid
+const main := do cps {
+  cps server = bind "127.0.0.1:8080";  -- Bind socket, store in 'server'
+  cps client = accept server;          -- Accept connection, store in 'client'
+  cps data = read client 1024;         -- Read data from client
+  cps _ = write_all client data;       -- Write data back (ignore result)
+  cps _ = close client;                -- Close connection
+  cps pass 0;                          -- Return 0
+}
+```
+
+## Error Handling
+
+Socket operations return errors when they fail. Errors are displayed with the IO error message:
+
+```
+error: IO error: Connection refused (os error 111):  @
+```
+
+Common errors:
+- `Connection refused` - No server listening on the target address
+- `Address already in use` - Port is already bound by another process
+- `Connection reset by peer` - Remote end closed the connection unexpectedly
+
+## Building a Simple Protocol
+
+Here's an example of a simple request/response protocol:
+
+```orchid
+import std::socket::tcp::(bind, accept, read, write_all, close)
+
+const handle_request := \request.
+  if request == "PING\n" then "PONG\n"
+  else if request == "TIME\n" then "2024-01-01 00:00:00\n"
+  else "UNKNOWN\n"
+
+const handle_client := \client. do cps {
+  cps request = read client 1024;
+  cps response = pass $ handle_request request;
+  cps _ = write_all client response;
+  cps _ = close client;
+  cps pass 0;
+}
+
+const server_loop := \server. do cps {
+  cps client = accept server;
+  cps _ = handle_client client;
+  cps pass $ server_loop server;
+}
+
+const main := do cps {
+  cps server = bind "127.0.0.1:9000";
+  cps pass $ server_loop server;
+}
+```
+
+## Limitations
+
+- The current implementation handles one client at a time (sequential accept loop)
+- Sockets are non-serializable (cannot be persisted or transferred across boundaries)
+- No UDP support (TCP only)
+- No TLS/SSL support
+
+## Troubleshooting
+
+### Port Already in Use
+
+If you get "Address already in use", either:
+1. Wait for the previous process to release the port
+2. Use a different port
+3. Kill the process using the port: `lsof -i :8080` then `kill <pid>`
+
+### Connection Refused
+
+Ensure the server is running before starting the client.
+
+### Build Errors
+
+Make sure to use the nightly toolchain:
+
+```sh
+rustup run nightly cargo build
+```

README.md

@@ -22,7 +22,7 @@ Namespaces are inspired by Rust modules and ES6. Every file and directory is imp
 The project uses both the stable and nightly rust toolchain. Run the examples with
 
 ```sh
-cargo orcx --release exec --proj ./examples/hello-world "src::main::main"
+cargo orcx -- exec --proj ./examples/hello-world "src::main::main"
 ```
 
 you can try modifying the examples, but error reporting for the time being is pretty terrible.
@@ -35,14 +35,14 @@ Orchids and mangrove trees form complex ecosystems; The flowers persuade the tre
 
 All contributions are welcome. For the time being, use the issue tracker to discuss ideas.
 
-Unless we agree on different terms, by contributing to this software you declare that you have created or otherwise have the right to license your contribution, agree to license it publicly under the general noncommercial licence included in this repository, and grant me (the owner of the project) a permanent, unrestricted license to use, modify, distribute and relicense your contribution. You retain ownership of your intellectual property to ensure that the copyleft protections cementing the noncommercial availability of the code are preserved.
+Unless we agree on different terms, by contributing to this software you declare that you have created or otherwise have the right to license your contribution, agree to license it publicly under the general noncommercial licence included in this repository, and grant me (the owner of the project) a permanent, unrestricted license to use, modify, distribute and relicense your contribution under any terms I see fit.
 
 ## About the license
 
 This software is free for noncommercial use. If you would like to use it for commercial purposes, or distribute your derivative work under a license that permits commercial use, contact me for a separate license. These licences are provided on a case-by-case basis with any limitations and compensation we agree on.
 
-I generally appreciate the ethos of free software, and particularly the patterns used in copyleft to cement the guarantees of the licence. However, I don't think commercial entities fit that ethos, and I think they should be addressed separately rather than attempting to ignore the inherent unfairness towards contributors.
+I generally appreciate the ethos of open-source software, and particularly the patterns used in copyleft to cement the longevity of the guarantees of the licence. However, I don't think commercial entities fit that ethos, and I think they should be addressed separately rather than attempting to ignore the inherent unfairness towards contributors.
 
-My intent with the custom license included in this project is to enable the strong guarantees of copyleft towards noncommercial users, while leaving commercial users to engage with this project and its possible future ecosystem in a commercial way; if you intend to profit off my work, the barest cash flow should justify shooting me an email and agreeing on a simple temporary profit sharing deal until you figure out your business model, and the cash flow of a full scale business should more than justify dedicated attention to the software you rely on.
+My intent with the custom license included in this project is to enable the strong guarantees of open-source towards open-source, while leaving commercial users to engage with this project and its possible future ecosystem in a commercial way; if you intend to profit off my work, the barest cash flow should justify shooting me an email and agreeing on a simple temporary profit sharing deal until you figure out your business model, and the cash flow of a full scale business should more than justify dedicated attention to the software you rely on.
 
 The clause about identifying marks is intended to prevent another pitfall of open-source, wherein Linux distros borrow entire codebases, break them, and then distribute the result under the original author's name. If would like to package Orchid, I'd be delighted if you would talk to me about making it official, but if you would rather operate independently, you should present your project as the rogue derivative work that it is rather than borrowing the original project's identity for something its owner has no control over.

async-fn-stream/Cargo.toml

@@ -7,4 +7,4 @@ edition = "2024"
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 
 [dev-dependencies]
-test_executors = "0.4.1"
+test_executors = "0.3.5"

examples/hello-world/main.orc

@@ -1,4 +1,2 @@
-let user = r[ "foo" 1, "bar" t[3, 4] ]
+let my_tuple = option::some t[1, 2]
-let _main = user.bar.1
+let main = tuple::get (option::expect my_tuple "tuple is none") 1
-
-let main = "foo" + string::slice "hello" 1 3 + "bar"

examples/http-server-simple/main.orc

@@ -0,0 +1,29 @@
+import std::socket::tcp::(bind, accept, read, write_all, close, peer_addr, listener_addr)
+import system::io::println
+
+let http_response = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 44\r\nConnection: close\r\n\r\n<html><body><h1>Hello Orchid!</h1></body></html>"
+
+let handle_client = \client. do cps {
+  cps addr = peer_addr client;
+  cps println $ "Client connected: ${addr}";
+  cps request = read client 4096;
+  cps println $ "Received request from ${addr}";
+  cps _ = write_all client http_response;
+  cps _ = close client;
+  cps println $ "Connection closed: ${addr}";
+  cps pass 0;
+}
+
+let accept_loop = \server. do cps {
+  cps client = accept server;
+  cps _ = handle_client client;
+  cps pass $ accept_loop server;
+}
+
+let main = do cps {
+  cps server = bind "127.0.0.1:8080";
+  cps addr = listener_addr server;
+  cps println $ "HTTP Server listening on ${addr}";
+  cps println "Press Ctrl+C to stop";
+  cps pass $ accept_loop server;
+}

examples/http-server/src/main.orc

@@ -0,0 +1,31 @@
+import std::socket::tcp::(bind, accept, read, write_all, close, peer_addr, listener_addr)
+import system::io::println
+
+cps println $ "Hello Orchid!";
+
+const http_response := "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 44\r\nConnection: close\r\n\r\n<html><body><h1>Hello Orchid!</h1></body></html>"
+
+const handle_client := \client. do cps {
+  cps addr = peer_addr client;
+  cps println $ "Client connected: ${addr}";
+  cps request = read client 4096;
+  cps println $ "Received request from ${addr}";
+  cps _ = write_all client http_response;
+  cps _ = close client;
+  cps println $ "Connection closed: ${addr}";
+  cps pass 0;
+}
+
+const accept_loop := \server. do cps {
+  cps client = accept server;
+  cps _ = handle_client client;
+  cps pass $ accept_loop server;
+}
+
+const main := do cps {
+  cps server = bind "127.0.0.1:8080";
+  cps addr = listener_addr server;
+  cps println $ "HTTP Server listening on ${addr}";
+  cps println "Press Ctrl+C to stop";
+  cps pass $ accept_loop server;
+}

examples/tcp-client/src/main.orc

@@ -0,0 +1,13 @@
+import std::socket::tcp::(connect, read, write_all, close, peer_addr)
+import system::io::println
+
+const main := do cps {
+  cps conn = connect "127.0.0.1:8080";
+  cps addr = peer_addr conn;
+  cps println $ "Connected to ${addr}";
+  cps _ = write_all conn "Hello from Orchid!\n";
+  cps response = read conn 1024;
+  cps println $ "Server response: ${response}";
+  cps _ = close conn;
+  cps pass 0;
+}

examples/tcp-echo-server/src/main.orc

@@ -0,0 +1,19 @@
+import std::socket::tcp::(bind, accept, read, write_all, close)
+
+const handle_client := \client. do cps {
+  cps data = read client 1024;
+  cps _ = write_all client data;
+  cps _ = close client;
+  cps pass 0;
+}
+
+const accept_loop := \server. do cps {
+  cps client = accept server;
+  cps _ = handle_client client;
+  cps pass $ accept_loop server;
+}
+
+const main := do cps {
+  cps server = bind "127.0.0.1:8080";
+  cps pass $ accept_loop server;
+}

orchid-api-derive/Cargo.toml

@@ -9,8 +9,8 @@ proc-macro = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-quote = "1.0.42"
+quote = "1.0.40"
-syn = { version = "2.0.112" }
+syn = { version = "2.0.106" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-proc-macro2 = "1.0.104"
+proc-macro2 = "1.0.101"
 itertools = "0.14.0"

orchid-api-derive/src/decode.rs

@@ -14,8 +14,8 @@ pub fn derive(input: TokenStream) -> TokenStream {
 		impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause {
 			async fn decode<R: orchid_api_traits::AsyncRead + ?Sized>(
 				mut read: std::pin::Pin<&mut R>
-			) -> std::io::Result<Self> {
+			) -> Self {
-				Ok(#decode)
+				#decode
 			}
 		}
 	};
@@ -30,7 +30,7 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream {
 				let syn::Field { ty, ident, .. } = &f;
 				quote! {
 					#ident : (Box::pin(< #ty as orchid_api_traits::Decode>::decode(read.as_mut()))
-						as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<_>>>>).await?
+						as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await
 				}
 			});
 			quote! { { #( #exprs, )* } }
@@ -40,7 +40,7 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream {
 				let ty = &field.ty;
 				quote! {
 					(Box::pin(< #ty as orchid_api_traits::Decode>::decode(read.as_mut()))
-						as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<_>>>>).await?,
+						as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await,
 				}
 			});
 			quote! { ( #( #exprs )* ) }
@@ -62,7 +62,7 @@ fn decode_body(data: &syn::Data) -> proc_macro2::TokenStream {
 				quote! { #id => Self::#ident #fields, }
 			});
 			quote! {
-				match <u8 as orchid_api_traits::Decode>::decode(read.as_mut()).await? {
+				match <u8 as orchid_api_traits::Decode>::decode(read.as_mut()).await {
 					#(#opts)*
 					x => panic!("Unrecognized enum kind {x}")
 				}

orchid-api-derive/src/encode.rs

@@ -17,9 +17,8 @@ pub fn derive(input: TokenStream) -> TokenStream {
 			async fn encode<W: orchid_api_traits::AsyncWrite + ?Sized>(
 				&self,
 				mut write: std::pin::Pin<&mut W>
-			) -> std::io::Result<()> {
+			) {
-				#encode;
+				#encode
-				Ok(())
 			}
 		}
 	};
@@ -44,7 +43,7 @@ fn encode_body(data: &syn::Data) -> Option<pm2::TokenStream> {
 				quote! {
 					Self::#ident #dest => {
 						(Box::pin((#i as u8).encode(write.as_mut()))
-							as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<()>>>>).await?;
+							as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await;
 						#body
 					}
 				}
@@ -62,7 +61,7 @@ fn encode_body(data: &syn::Data) -> Option<pm2::TokenStream> {
 fn encode_names<T: ToTokens>(names: impl Iterator<Item = T>) -> pm2::TokenStream {
 	quote! { #(
 		(Box::pin(#names .encode(write.as_mut()))
-			as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<()>>>>).await?;
+			as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await;
 	)* }
 }
 

orchid-api-derive/src/hierarchy.rs

@@ -120,3 +120,6 @@ fn get_ancestry(input: &DeriveInput) -> Option<Vec<pm2::TokenStream>> {
 fn is_extendable(input: &DeriveInput) -> bool {
 	input.attrs.iter().any(|a| a.path().get_ident().is_some_and(|i| *i == "extendable"))
 }
+
+#[test]
+fn test_wtf() { eprintln!("{}", gen_casts(&[quote!(ExtHostReq)], &quote!(BogusReq))) }

orchid-api-traits/Cargo.toml

@@ -6,7 +6,8 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 itertools = "0.14.0"
 never = "0.1.0"
-ordered-float = "5.1.0"
+ordered-float = "5.0.0"

orchid-api-traits/src/coding.rs

@@ -1,44 +1,33 @@
 use std::collections::HashMap;
 use std::future::Future;
 use std::hash::Hash;
-use std::io;
 use std::num::NonZero;
 use std::ops::{Range, RangeInclusive};
 use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 
-use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use async_fn_stream::stream;
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, StreamExt};
 use never::Never;
 use ordered_float::NotNan;
 
-use crate::{decode_err, decode_err_for, encode_enum, spin_on};
+use crate::encode_enum;
 
-pub trait Decode: 'static + Sized {
+pub trait Decode: 'static {
 	/// Decode an instance from the beginning of the buffer. Return the decoded
 	/// data and the remaining buffer.
-	fn decode<R: AsyncRead + ?Sized>(
+	fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
-		read: Pin<&mut R>,
-	) -> impl Future<Output = io::Result<Self>> + '_;
-	fn decode_slice(slc: &mut &[u8]) -> Self {
-		spin_on(Self::decode(Pin::new(slc) as Pin<&mut _>)).expect("Decode from slice cannot fail")
-	}
 }
 pub trait Encode {
 	/// Append an instance of the struct to the buffer
-	fn encode<W: AsyncWrite + ?Sized>(
+	fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
-		&self,
-		write: Pin<&mut W>,
-	) -> impl Future<Output = io::Result<()>>;
-	fn encode_vec(&self, vec: &mut Vec<u8>) {
-		spin_on(self.encode(Pin::new(vec) as Pin<&mut _>)).expect("Encode to vector cannot fail")
-	}
 }
 pub trait Coding: Encode + Decode + Clone {
-	fn get_decoder<T: 'static>(
+	fn get_decoder<T: 'static, F: Future<Output = T> + 'static>(
-		map: impl AsyncFn(Self) -> T + Clone + 'static,
+		map: impl Fn(Self) -> F + Clone + 'static,
-	) -> impl AsyncFn(Pin<&mut dyn AsyncRead>) -> io::Result<T> {
+	) -> impl AsyncFn(Pin<&mut dyn AsyncRead>) -> T {
-		async move |r| Ok(map(Self::decode(r).await?).await)
+		async move |r| map(Self::decode(r).await).await
 	}
 }
 impl<T: Encode + Decode + Clone> Coding for T {}
@@ -46,15 +35,15 @@ impl<T: Encode + Decode + Clone> Coding for T {}
 macro_rules! num_impl {
 	($number:ty) => {
 		impl Decode for $number {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; (<$number>::BITS / 8) as usize];
-				read.read_exact(&mut bytes).await?;
+				read.read_exact(&mut bytes).await.unwrap();
-				Ok(<$number>::from_be_bytes(bytes))
+				<$number>::from_be_bytes(bytes)
 			}
 		}
 		impl Encode for $number {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-				write.write_all(&self.to_be_bytes()).await
+				write.write_all(&self.to_be_bytes()).await.expect("Could not write number")
 			}
 		}
 	};
@@ -73,12 +62,12 @@ num_impl!(i8);
 macro_rules! nonzero_impl {
 	($name:ty) => {
 		impl Decode for NonZero<$name> {
-			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> io::Result<Self> {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
-				Self::new(<$name as Decode>::decode(read).await?).ok_or_else(decode_err)
+				Self::new(<$name as Decode>::decode(read).await).unwrap()
 			}
 		}
 		impl Encode for NonZero<$name> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 				self.get().encode(write).await
 			}
 		}
@@ -97,22 +86,22 @@ nonzero_impl!(i64);
 nonzero_impl!(i128);
 
 impl<T: Encode + ?Sized> Encode for &T {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		(**self).encode(write).await
 	}
 }
 macro_rules! float_impl {
 	($t:ty, $size:expr) => {
 		impl Decode for NotNan<$t> {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; $size];
-				read.read_exact(&mut bytes).await?;
+				read.read_exact(&mut bytes).await.unwrap();
-				NotNan::new(<$t>::from_be_bytes(bytes)).map_err(|_| decode_err())
+				NotNan::new(<$t>::from_be_bytes(bytes)).expect("Float was NaN")
 			}
 		}
 		impl Encode for NotNan<$t> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-				write.write_all(&self.as_ref().to_be_bytes()).await
+				write.write_all(&self.as_ref().to_be_bytes()).await.expect("Could not write number")
 			}
 		}
 	};
@@ -122,77 +111,78 @@ float_impl!(f64, 8);
 float_impl!(f32, 4);
 
 impl Decode for String {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len: usize = u64::decode(read.as_mut()).await?.try_into().map_err(decode_err_for)?;
+		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
 		let mut data = vec![0u8; len];
-		read.read_exact(&mut data).await?;
+		read.read_exact(&mut data).await.unwrap();
-		Ok(std::str::from_utf8(&data).map_err(decode_err_for)?.to_owned())
+		std::str::from_utf8(&data).expect("String invalid UTF-8").to_owned()
 	}
 }
 impl Encode for String {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).map_err(decode_err_for)?.encode(write.as_mut()).await?;
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		write.write_all(self.as_bytes()).await
+		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl Encode for str {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).map_err(decode_err_for)?.encode(write.as_mut()).await?;
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		write.write_all(self.as_bytes()).await
+		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl<T: Decode> Decode for Vec<T> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len = u64::decode(read.as_mut()).await?;
+		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		let mut values = Vec::with_capacity(len.try_into().map_err(decode_err_for)?);
+		stream(async |mut cx| {
 			for _ in 0..len {
-			values.push(T::decode(read.as_mut()).await?);
+				cx.emit(T::decode(read.as_mut()).await).await
 			}
-		Ok(values)
+		})
+		.collect()
+		.await
 	}
 }
 impl<T: Encode> Encode for Vec<T> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		self.as_slice().encode(write).await
 	}
 }
 impl<T: Encode> Encode for [T] {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await?;
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		for t in self.iter() {
-			t.encode(write.as_mut()).await?
+			t.encode(write.as_mut()).await
 		}
-		Ok(())
 	}
 }
 impl<T: Decode> Decode for Option<T> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		Ok(match bool::decode(read.as_mut()).await? {
+		match u8::decode(read.as_mut()).await {
-			false => None,
+			0 => None,
-			true => Some(T::decode(read).await?),
+			1 => Some(T::decode(read).await),
-		})
+			x => panic!("{x} is not a valid option value"),
+		}
 	}
 }
 impl<T: Encode> Encode for Option<T> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		self.is_some().encode(write.as_mut()).await?;
+		let t = if let Some(t) = self { t } else { return 0u8.encode(write.as_mut()).await };
-		if let Some(t) = self {
+		1u8.encode(write.as_mut()).await;
-			t.encode(write).await?
+		t.encode(write).await;
-		}
-		Ok(())
 	}
 }
 impl<T: Decode, E: Decode> Decode for Result<T, E> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		Ok(match bool::decode(read.as_mut()).await? {
+		match u8::decode(read.as_mut()).await {
-			false => Self::Ok(T::decode(read).await?),
+			0 => Self::Ok(T::decode(read).await),
-			true => Self::Err(E::decode(read).await?),
+			1 => Self::Err(E::decode(read).await),
-		})
+			x => panic!("Invalid Result tag {x}"),
+		}
 	}
 }
 
 impl<T: Encode, E: Encode> Encode for Result<T, E> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		match self {
 			Ok(t) => encode_enum(write, 0, |w| t.encode(w)).await,
 			Err(e) => encode_enum(write, 1, |w| e.encode(w)).await,
@@ -200,37 +190,36 @@ impl<T: Encode, E: Encode> Encode for Result<T, E> {
 	}
 }
 impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len = u64::decode(read.as_mut()).await?;
+		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		let mut map = HashMap::with_capacity(len.try_into().map_err(decode_err_for)?);
+		stream(async |mut cx| {
 			for _ in 0..len {
-			map.insert(K::decode(read.as_mut()).await?, V::decode(read.as_mut()).await?);
+				cx.emit(<(K, V)>::decode(read.as_mut()).await).await
 			}
-		Ok(map)
+		})
+		.collect()
+		.await
 	}
 }
 impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await?;
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		for (key, value) in self.iter() {
+		for pair in self.iter() {
-			key.encode(write.as_mut()).await?;
+			pair.encode(write.as_mut()).await
-			value.encode(write.as_mut()).await?;
 		}
-		Ok(())
 	}
 }
 macro_rules! tuple {
   (($($t:ident)*) ($($T:ident)*)) => {
     impl<$($T: Decode),*> Decode for ($($T,)*) {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-				Ok(($($T::decode(read.as_mut()).await?,)*))
+				($($T::decode(read.as_mut()).await,)*)
 			}
     }
     impl<$($T: Encode),*> Encode for ($($T,)*) {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let ($($t,)*) = self;
-        $( $t.encode(write.as_mut()).await?; )*
+        $( $t.encode(write.as_mut()).await; )*
-				Ok(())
       }
     }
   };
@@ -254,67 +243,63 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I));
 tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16
 
 impl Decode for () {
-	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> io::Result<Self> { Ok(()) }
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {}
 }
 impl Encode for () {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) -> io::Result<()> { Ok(()) }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) {}
 }
 impl Decode for Never {
-	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {
 		unreachable!("A value of Never cannot exist so it can't have been serialized");
 	}
 }
 impl Encode for Never {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
-		match *self {}
-	}
 }
 impl Decode for bool {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let mut buf = [0];
-		read.read_exact(&mut buf).await?;
+		read.read_exact(&mut buf).await.unwrap();
-		Ok(buf[0] != 0)
+		buf[0] != 0
 	}
 }
 impl Encode for bool {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await
+		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await.unwrap()
 	}
 }
 impl<T: Decode, const N: usize> Decode for [T; N] {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let mut v = Vec::with_capacity(N);
+		let v = stream(async |mut cx| {
 			for _ in 0..N {
-			v.push(T::decode(read.as_mut()).await?);
+				cx.emit(T::decode(read.as_mut()).await).await
-		}
-		match v.try_into() {
-			Err(_) => unreachable!("The length of this stream is statically known"),
-			Ok(arr) => Ok(arr),
 			}
+		})
+		.collect::<Vec<_>>()
+		.await;
+		v.try_into().unwrap_or_else(|_| unreachable!("The length of this stream is statically known"))
 	}
 }
 impl<T: Encode, const N: usize> Encode for [T; N] {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		for t in self.iter() {
-			t.encode(write.as_mut()).await?
+			t.encode(write.as_mut()).await
 		}
-		Ok(())
 	}
 }
 
 macro_rules! two_end_range {
   ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => {
     impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-				Ok(T::decode(read.as_mut()).await? $op T::decode(read).await?)
+				T::decode(read.as_mut()).await $op T::decode(read).await
 			}
     }
     impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let $this = self;
-        ($start).encode(write.as_mut()).await?;
+        ($start).encode(write.as_mut()).await;
-        ($end).encode(write).await?;
+        ($end).encode(write).await;
-				Ok(())
       }
     }
   }
@@ -326,12 +311,12 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end());
 macro_rules! smart_ptr {
 	($name:tt) => {
 		impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> io::Result<Self> {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
-				Ok($name::new(T::decode(read).await?))
+				$name::new(T::decode(read).await)
 			}
 		}
 		impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 				(**self).encode(write).await
 			}
 		}
@@ -343,12 +328,12 @@ smart_ptr!(Rc);
 smart_ptr!(Box);
 
 impl Decode for char {
-	async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
-		char::from_u32(u32::decode(read).await?).ok_or_else(decode_err)
+		char::from_u32(u32::decode(read).await).unwrap()
 	}
 }
 impl Encode for char {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		(*self as u32).encode(write).await
 	}
 }

orchid-api-traits/src/helpers.rs

@@ -1,24 +1,24 @@
-use std::error::Error;
+use std::future::Future;
-use std::io;
+use std::pin::Pin;
-use std::pin::{Pin, pin};
-use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::task::{Context, Poll, Wake};
 
-use futures::{AsyncRead, AsyncReadExt, AsyncWrite};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use itertools::{Chunk, Itertools};
 
 use crate::Encode;
 
-pub async fn encode_enum<'a, W: AsyncWrite + ?Sized>(
+pub async fn encode_enum<'a, W: AsyncWrite + ?Sized, F: Future<Output = ()>>(
 	mut write: Pin<&'a mut W>,
 	id: u8,
-	f: impl AsyncFnOnce(Pin<&'a mut W>) -> io::Result<()>,
+	f: impl FnOnce(Pin<&'a mut W>) -> F,
-) -> io::Result<()> {
+) {
-	id.encode(write.as_mut()).await?;
+	id.encode(write.as_mut()).await;
 	f(write).await
 }
 
+pub async fn write_exact<W: AsyncWrite + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
+	write.write_all(bytes).await.expect("Failed to write exact bytes")
+}
+
 pub fn print_bytes(b: &[u8]) -> String {
 	(b.iter().map(|b| format!("{b:02x}")))
 		.chunks(4)
@@ -27,52 +27,16 @@ pub fn print_bytes(b: &[u8]) -> String {
 		.join("  ")
 }
 
-pub async fn read_exact<R: AsyncRead + ?Sized>(
+pub async fn read_exact<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
-	mut read: Pin<&mut R>,
-	bytes: &'static [u8],
-) -> io::Result<()> {
 	let mut data = vec![0u8; bytes.len()];
-	read.read_exact(&mut data).await?;
+	read.read_exact(&mut data).await.expect("Failed to read bytes");
-	if data == bytes {
+	if data != bytes {
-		Ok(())
+		panic!("Wrong bytes!\nExpected: {}\nFound: {}", print_bytes(bytes), print_bytes(&data));
-	} else {
-		let msg =
-			format!("Wrong bytes!\nExpected: {}\nFound: {}", print_bytes(bytes), print_bytes(&data));
-		Err(io::Error::new(io::ErrorKind::InvalidData, msg))
 	}
 }
 
-pub fn enc_vec(enc: &impl Encode) -> Vec<u8> {
+pub async fn enc_vec(enc: &impl Encode) -> Vec<u8> {
 	let mut vec = Vec::new();
-	enc.encode_vec(&mut vec);
+	enc.encode(Pin::new(&mut vec)).await;
 	vec
 }
-
-/// Raises a bool flag when called
-struct FlagWaker(AtomicBool);
-impl Wake for FlagWaker {
-	fn wake(self: Arc<Self>) { self.0.store(true, Ordering::Relaxed) }
-}
-
-pub fn spin_on<F: Future>(fut: F) -> F::Output {
-	let flag = AtomicBool::new(false);
-	let flag_waker = Arc::new(FlagWaker(flag));
-	let mut future = pin!(fut);
-	loop {
-		let waker = flag_waker.clone().into();
-		let mut ctx = Context::from_waker(&waker);
-		match future.as_mut().poll(&mut ctx) {
-			// ideally the future should return synchronously
-			Poll::Ready(res) => break res,
-			// poorly written futures may yield and immediately wake
-			Poll::Pending if flag_waker.0.load(Ordering::Relaxed) => (),
-			// there is no external event to wait for, this has to be a deadlock
-			Poll::Pending => panic!("Future inside spin_on cannot block"),
-		};
-	}
-}
-
-pub fn decode_err() -> io::Error { io::Error::new(io::ErrorKind::InvalidData, "Unexpected zero") }
-pub fn decode_err_for(e: impl Error) -> io::Error {
-	io::Error::new(io::ErrorKind::InvalidData, e.to_string())
-}

orchid-api-traits/src/hierarchy.rs

@@ -29,21 +29,25 @@ pub trait Extends: InHierarchy<IsRoot = TLFalse> + Into<Self::Parent> {
 pub trait UnderRootImpl<IsRoot: TLBool>: Sized {
 	type __Root: UnderRoot<IsRoot = TLTrue, Root = Self::__Root>;
 	fn __into_root(self) -> Self::__Root;
+	fn __try_from_root(root: Self::__Root) -> Result<Self, Self::__Root>;
 }
 
 pub trait UnderRoot: InHierarchy {
 	type Root: UnderRoot<IsRoot = TLTrue, Root = Self::Root>;
 	fn into_root(self) -> Self::Root;
+	fn try_from_root(root: Self::Root) -> Result<Self, Self::Root>;
 }
 
 impl<T: InHierarchy + UnderRootImpl<T::IsRoot>> UnderRoot for T {
 	type Root = <Self as UnderRootImpl<<Self as InHierarchy>::IsRoot>>::__Root;
 	fn into_root(self) -> Self::Root { self.__into_root() }
+	fn try_from_root(root: Self::Root) -> Result<Self, Self::Root> { Self::__try_from_root(root) }
 }
 
 impl<T: InHierarchy<IsRoot = TLTrue>> UnderRootImpl<TLTrue> for T {
 	type __Root = Self;
 	fn __into_root(self) -> Self::__Root { self }
+	fn __try_from_root(root: Self::__Root) -> Result<Self, Self::__Root> { Ok(root) }
 }
 
 impl<T: InHierarchy<IsRoot = TLFalse> + Extends> UnderRootImpl<TLFalse> for T {
@@ -53,4 +57,8 @@ impl<T: InHierarchy<IsRoot = TLFalse> + Extends> UnderRootImpl<TLFalse> for T {
 	fn __into_root(self) -> Self::__Root {
 		<Self as Into<<Self as Extends>::Parent>>::into(self).into_root()
 	}
+	fn __try_from_root(root: Self::__Root) -> Result<Self, Self::__Root> {
+		let parent = <Self as Extends>::Parent::try_from_root(root)?;
+		parent.clone().try_into().map_err(|_| parent.into_root())
+	}
 }

orchid-api-traits/src/relations.rs

@@ -1,6 +1,5 @@
 use core::fmt;
-
+use std::future::Future;
-use never::Never;
 
 use super::coding::Coding;
 use crate::helpers::enc_vec;
@@ -9,22 +8,20 @@ pub trait Request: fmt::Debug + Sized + 'static {
 	type Response: fmt::Debug + Coding + 'static;
 }
 
-pub fn respond<R: Request>(_: &R, rep: R::Response) -> Vec<u8> { enc_vec(&rep) }
+pub async fn respond<R: Request>(_: &R, rep: R::Response) -> Vec<u8> { enc_vec(&rep).await }
+pub async fn respond_with<R: Request, F: Future<Output = R::Response>>(
+	r: &R,
+	f: impl FnOnce(&R) -> F,
+) -> Vec<u8> {
+	respond(r, f(r).await).await
+}
 
 pub trait Channel: 'static {
 	type Req: Coding + Sized + 'static;
 	type Notif: Coding + Sized + 'static;
 }
-impl Channel for Never {
-	type Notif = Never;
-	type Req = Never;
-}
 
 pub trait MsgSet: Sync + 'static {
 	type In: Channel;
 	type Out: Channel;
 }
-impl MsgSet for Never {
-	type In = Never;
-	type Out = Never;
-}

orchid-api/Cargo.toml

@@ -6,12 +6,11 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ordered-float = "5.1.0"
+ordered-float = "5.0.0"
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 itertools = "0.14.0"
-unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
 
 [dev-dependencies]
-test_executors = "0.4.1"
+test_executors = "0.3.5"

orchid-api/src/binary.rs

@@ -1,90 +0,0 @@
-//! # Binary extension definition
-//!
-//! A binary extension is a DLL / shared object / dylib with a symbol called
-//! `orchid_extension_main` which accepts a single argument of type
-//! [ExtensionContext]. Once that is received, communication continuees through
-//! the channel with the same protocol outlined in [crate::proto]
-
-use unsync_pipe::{Reader, Writer};
-
-/// !Send !Sync owned waker
-///
-/// This object is [Clone] for convenience but it has `drop` and no `clone` so
-/// interactions must reflect a single logical owner
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub struct OwnedWakerBin {
-	pub data: *const (),
-	/// `self`
-	pub drop: extern "C" fn(*const ()),
-	/// `self`
-	pub wake: extern "C" fn(*const ()),
-	/// `&self`
-	pub wake_ref: extern "C" fn(*const ()),
-}
-
-/// !Send !Sync, equivalent to `&mut Context<'a>`, hence no `drop`.
-/// When received in [FutureBin::poll], it must not outlive the call.
-///
-/// You cannot directly wake using this waker, because such a trampoline would
-/// pass through the binary interface twice for no reason. An efficient
-/// implementation should implement that trampoline action internally, whereas
-/// an inefficient but compliant implementation can clone a fresh waker and use
-/// it up.
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub struct FutureContextBin {
-	pub data: *const (),
-	/// `&self`
-	pub waker: extern "C" fn(*const ()) -> OwnedWakerBin,
-}
-
-/// ABI-stable `Poll<()>`
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub enum UnitPoll {
-	Pending,
-	Ready,
-}
-
-/// ABI-stable `Pin<Box<dyn Future<Output = ()>>>`
-///
-/// This object is [Clone] for convenience, but it has `drop` and no `clone` so
-/// interactions must reflect a single logical owner
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub struct FutureBin {
-	pub data: *const (),
-	/// `self`
-	pub drop: extern "C" fn(*const ()),
-	/// `&mut self` Equivalent to [Future::poll]
-	pub poll: extern "C" fn(*const (), FutureContextBin) -> UnitPoll,
-}
-
-/// Handle for a runtime that allows its holder to spawn futures across dynamic
-/// library boundaries
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub struct SpawnerBin {
-	pub data: *const (),
-	/// `self`
-	pub drop: extern "C" fn(*const ()),
-	/// `&self` Add a future to this extension's task
-	pub spawn: extern "C" fn(*const (), FutureBin),
-}
-
-/// Extension context.
-///
-/// This struct is a plain old value, all of the contained values have a
-/// distinct `drop` member
-#[repr(C)]
-pub struct ExtensionContext {
-	/// Spawns tasks associated with this extension
-	pub spawner: SpawnerBin,
-	/// serialized [crate::HostExtChannel]
-	pub input: Reader,
-	/// serialized [crate::ExtHostChannel]
-	pub output: Writer,
-	/// UTF-8 log stream directly to log service.
-	pub log: Writer,
-}

orchid-api/src/interner.rs

@@ -3,7 +3,7 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{ExtHostNotif, ExtHostReq, HostExtReq};
+use crate::{ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
@@ -71,21 +71,18 @@ pub struct TStr(pub NonZeroU64);
 pub struct TStrv(pub NonZeroU64);
 
 /// A request to sweep the replica. The master will not be sweeped until all
-/// replicas respond. For efficiency, replicas should make sure to send the
+/// replicas respond, as it must retain everything the replicas retained
-/// [Sweeped] notif before returning.
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct Sweep;
 impl Request for Sweep {
-	type Response = ();
+	type Response = Retained;
 }
 
-/// List of keys in this replica that were removed during a sweep. This may have
+/// List of keys in this replica that couldn't be sweeped because local
-/// been initiated via a [Sweep] request, but can also be triggered by the
+/// datastructures reference their value.
-/// replica autonomously.
+#[derive(Clone, Debug, Coding)]
-#[derive(Clone, Debug, Coding, Hierarchy)]
+pub struct Retained {
-#[extends(ExtHostNotif)]
-pub struct Sweeped {
 	pub strings: Vec<TStr>,
 	pub vecs: Vec<TStrv>,
 }

orchid-api/src/lexer.rs

@@ -28,7 +28,7 @@ impl Request for LexExpr {
 #[derive(Clone, Debug, Coding)]
 pub struct LexedExpr {
 	pub pos: u32,
-	pub expr: Vec<TokenTree>,
+	pub expr: TokenTree,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/lib.rs

@@ -1,4 +1,3 @@
-pub mod binary;
 mod lexer;
 pub use lexer::*;
 mod format;

orchid-api/src/logging.rs

@@ -1,30 +1,14 @@
-use std::collections::HashMap;
-
 use orchid_api_derive::{Coding, Hierarchy};
 
-use crate::{ExtHostNotif, TStr};
+use crate::ExtHostNotif;
 
-/// Describes what to do with a log stream.
-/// Log streams are unstructured utf8 text unless otherwise stated.
 #[derive(Clone, Debug, Coding, PartialEq, Eq, Hash)]
 pub enum LogStrategy {
-	/// Context-dependent default stream, often stderr
+	StdErr,
-	Default,
+	File(String),
-	/// A file on the local filesystem
-	File { path: String, append: bool },
-	/// Discard any log output
 	Discard,
 }
 
-#[derive(Clone, Debug, Coding)]
-pub struct Logger {
-	pub routing: HashMap<String, LogStrategy>,
-	pub default: Option<LogStrategy>,
-}
-
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostNotif)]
-pub struct Log {
+pub struct Log(pub String);
-	pub category: TStr,
-	pub message: String,
-}

orchid-api/src/proto.rs

@@ -22,49 +22,51 @@
 //! be preserved. Toolkits must ensure that the client code is able to observe
 //! the ordering of messages.
 
-use std::io;
 use std::pin::Pin;
 
-use futures::{AsyncRead, AsyncWrite, AsyncWriteExt};
+use futures::{AsyncRead, AsyncWrite};
 use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact};
+use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
 
-use crate::{Sweeped, atom, expr, interner, lexer, logging, parser, system, tree};
+use crate::{atom, expr, interner, lexer, logging, parser, system, tree};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
-#[derive(Clone, Debug)]
 pub struct HostHeader {
-	pub logger: logging::Logger,
+	pub log_strategy: logging::LogStrategy,
+	pub msg_logs: logging::LogStrategy,
 }
 impl Decode for HostHeader {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		read_exact(read.as_mut(), HOST_INTRO).await?;
+		read_exact(read.as_mut(), HOST_INTRO).await;
-		Ok(Self { logger: logging::Logger::decode(read).await? })
+		Self {
+			log_strategy: logging::LogStrategy::decode(read.as_mut()).await,
+			msg_logs: logging::LogStrategy::decode(read.as_mut()).await,
+		}
 	}
 }
 impl Encode for HostHeader {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write.write_all(HOST_INTRO).await?;
+		write_exact(write.as_mut(), HOST_INTRO).await;
-		self.logger.encode(write.as_mut()).await
+		self.log_strategy.encode(write.as_mut()).await;
+		self.msg_logs.encode(write.as_mut()).await
 	}
 }
 
 static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
-#[derive(Clone, Debug)]
 pub struct ExtensionHeader {
 	pub name: String,
 	pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		read_exact(read.as_mut(), EXT_INTRO).await?;
+		read_exact(read.as_mut(), EXT_INTRO).await;
-		Ok(Self { name: String::decode(read.as_mut()).await?, systems: Vec::decode(read).await? })
+		Self { name: String::decode(read.as_mut()).await, systems: Vec::decode(read).await }
 	}
 }
 impl Encode for ExtensionHeader {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write.write_all(EXT_INTRO).await?;
+		write_exact(write.as_mut(), EXT_INTRO).await;
-		self.name.encode(write.as_mut()).await?;
+		self.name.encode(write.as_mut()).await;
 		self.systems.encode(write).await
 	}
 }
@@ -97,7 +99,6 @@ pub enum ExtHostReq {
 pub enum ExtHostNotif {
 	ExprNotif(expr::ExprNotif),
 	Log(logging::Log),
-	Sweeped(Sweeped),
 }
 
 pub struct ExtHostChannel;
@@ -154,22 +155,22 @@ impl MsgSet for HostMsgSet {
 
 #[cfg(test)]
 mod tests {
-	use std::collections::HashMap;
-
 	use orchid_api_traits::enc_vec;
 	use ordered_float::NotNan;
 	use test_executors::spin_on;
 
 	use super::*;
-	use crate::Logger;
 
 	#[test]
 	fn host_header_enc() {
 		spin_on(async {
-			let hh = HostHeader { logger: Logger { routing: HashMap::new(), default: None } };
+			let hh = HostHeader {
-			let mut enc = &enc_vec(&hh)[..];
+				log_strategy: logging::LogStrategy::File("SomeFile".to_string()),
+				msg_logs: logging::LogStrategy::File("SomeFile".to_string()),
+			};
+			let mut enc = &enc_vec(&hh).await[..];
 			eprintln!("Encoded to {enc:?}");
-			HostHeader::decode(Pin::new(&mut enc)).await.unwrap();
+			HostHeader::decode(Pin::new(&mut enc)).await;
 			assert_eq!(enc, []);
 		})
 	}
@@ -186,9 +187,9 @@ mod tests {
 					priority: NotNan::new(1f64).unwrap(),
 				}],
 			};
-			let mut enc = &enc_vec(&eh)[..];
+			let mut enc = &enc_vec(&eh).await[..];
 			eprintln!("Encoded to {enc:?}");
-			ExtensionHeader::decode(Pin::new(&mut enc)).await.unwrap();
+			ExtensionHeader::decode(Pin::new(&mut enc)).await;
 			assert_eq!(enc, [])
 		})
 	}

orchid-api/src/tree.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 use std::fmt;
 use std::num::NonZeroU64;
 use std::ops::Range;
+use std::rc::Rc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
@@ -47,7 +48,7 @@ pub enum Token {
 	/// NewExpr(Bottom) because it fails in dead branches too.
 	Bottom(Vec<OrcError>),
 	/// A comment
-	Comment(TStr),
+	Comment(Rc<String>),
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]

orchid-base/Cargo.toml

@@ -6,17 +6,12 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
-bound = "0.6.0"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.20"
-futures = { version = "0.3.31", default-features = false, features = [
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-	"std",
+hashbrown = "0.16.0"
-	"async-await",
-] }
-hashbrown = "0.16.1"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
 never = "0.1.0"
@@ -24,12 +19,10 @@ num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-ordered-float = "5.1.0"
+ordered-float = "5.0.0"
-regex = "1.12.2"
+regex = "1.11.2"
-rust-embed = "8.9.0"
+rust-embed = "8.7.2"
+some_executor = "0.6.1"
 substack = "1.1.1"
+test_executors = "0.3.5"
 trait-set = "0.3.0"
-task-local = "0.1.0"
-
-[dev-dependencies]
-futures = "0.3.31"

orchid-base/src/binary.rs

@@ -1,121 +0,0 @@
-use std::mem;
-use std::pin::Pin;
-use std::rc::Rc;
-use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker};
-
-use crate::api;
-
-type WideBox = Box<dyn Future<Output = ()>>;
-
-static OWNED_VTABLE: RawWakerVTable = RawWakerVTable::new(
-	|data| {
-		let data = unsafe { Rc::<api::binary::OwnedWakerBin>::from_raw(data as *const _) };
-		let val = RawWaker::new(Rc::into_raw(data.clone()) as *const (), &OWNED_VTABLE);
-		// Clone must create a duplicate of the Rc, so it has to be un-leaked, cloned,
-		// then leaked again.
-		let _ = Rc::into_raw(data);
-		val
-	},
-	|data| {
-		// Wake must awaken the task and then clean up the state, so the waker must be
-		// un-leaked
-		let data = unsafe { Rc::<api::binary::OwnedWakerBin>::from_raw(data as *const _) };
-		(data.wake)(data.data);
-		mem::drop(data);
-	},
-	|data| {
-		// Wake-by-ref must awaken the task while preserving the future, so the Rc is
-		// untouched
-		let data = unsafe { (data as *const api::binary::OwnedWakerBin).as_ref() }.unwrap();
-		(data.wake_ref)(data.data);
-	},
-	|data| {
-		// Drop must clean up the state, so the waker must be un-leaked
-		let data = unsafe { Rc::<api::binary::OwnedWakerBin>::from_raw(data as *const _) };
-		(data.drop)(data.data);
-		mem::drop(data);
-	},
-);
-
-struct BorrowedWakerData<'a> {
-	go_around: &'a mut bool,
-	cx: api::binary::FutureContextBin,
-}
-static BORROWED_VTABLE: RawWakerVTable = RawWakerVTable::new(
-	|data| {
-		let data = unsafe { (data as *mut BorrowedWakerData).as_mut() }.unwrap();
-		let owned_data = Rc::<api::binary::OwnedWakerBin>::new((data.cx.waker)(data.cx.data));
-		RawWaker::new(Rc::into_raw(owned_data) as *const (), &OWNED_VTABLE)
-	},
-	|data| *unsafe { (data as *mut BorrowedWakerData).as_mut() }.unwrap().go_around = true,
-	|data| *unsafe { (data as *mut BorrowedWakerData).as_mut() }.unwrap().go_around = true,
-	|_data| {},
-);
-
-/// Convert a future to a binary-compatible format that can be sent across
-/// dynamic library boundaries
-pub fn future_to_vt<Fut: Future<Output = ()> + 'static>(fut: Fut) -> api::binary::FutureBin {
-	let wide_box = Box::new(fut) as WideBox;
-	let data = Box::into_raw(Box::new(wide_box));
-	extern "C" fn drop(raw: *const ()) {
-		mem::drop(unsafe { Box::<WideBox>::from_raw(raw as *mut _) })
-	}
-	extern "C" fn poll(raw: *const (), cx: api::binary::FutureContextBin) -> api::binary::UnitPoll {
-		let mut this = unsafe { Pin::new_unchecked(&mut **(raw as *mut WideBox).as_mut().unwrap()) };
-		loop {
-			let mut go_around = false;
-			let borrowed_waker = unsafe {
-				Waker::from_raw(RawWaker::new(
-					&mut BorrowedWakerData { go_around: &mut go_around, cx } as *mut _ as *const (),
-					&BORROWED_VTABLE,
-				))
-			};
-			let mut ctx = Context::from_waker(&borrowed_waker);
-			let result = this.as_mut().poll(&mut ctx);
-			if matches!(result, Poll::Ready(())) {
-				break api::binary::UnitPoll::Ready;
-			}
-			if !go_around {
-				break api::binary::UnitPoll::Pending;
-			}
-		}
-	}
-	api::binary::FutureBin { data: data as *const _, drop, poll }
-}
-
-struct VirtualFuture {
-	vt: api::binary::FutureBin,
-}
-impl Unpin for VirtualFuture {}
-impl Future for VirtualFuture {
-	type Output = ();
-	fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-		extern "C" fn waker(raw: *const ()) -> api::binary::OwnedWakerBin {
-			let waker = unsafe { (raw as *mut Context).as_mut() }.unwrap().waker().clone();
-			let data = Box::into_raw(Box::<Waker>::new(waker)) as *const ();
-			return api::binary::OwnedWakerBin { data, drop, wake, wake_ref };
-			extern "C" fn drop(raw: *const ()) {
-				mem::drop(unsafe { Box::<Waker>::from_raw(raw as *mut Waker) })
-			}
-			extern "C" fn wake(raw: *const ()) {
-				unsafe { Box::<Waker>::from_raw(raw as *mut Waker) }.wake();
-			}
-			extern "C" fn wake_ref(raw: *const ()) {
-				unsafe { (raw as *mut Waker).as_mut() }.unwrap().wake_by_ref();
-			}
-		}
-		let cx = api::binary::FutureContextBin { data: cx as *mut Context as *const (), waker };
-		let result = (self.vt.poll)(self.vt.data, cx);
-		match result {
-			api::binary::UnitPoll::Pending => Poll::Pending,
-			api::binary::UnitPoll::Ready => Poll::Ready(()),
-		}
-	}
-}
-impl Drop for VirtualFuture {
-	fn drop(&mut self) { (self.vt.drop)(self.vt.data) }
-}
-
-/// Receive a future sent across dynamic library boundaries and convert it into
-/// an owned object
-pub fn vt_to_future(vt: api::binary::FutureBin) -> impl Future<Output = ()> { VirtualFuture { vt } }

orchid-base/src/builtin.rs

@@ -0,0 +1,34 @@
+use std::ops::Deref;
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+
+use crate::api;
+
+pub type Spawner = Rc<dyn Fn(LocalBoxFuture<'static, ()>)>;
+
+/// The 3 primary contact points with an extension are
+/// - send a message
+/// - wait for a message to arrive
+/// - wait for the extension to stop after exit (this is the implicit Drop)
+///
+/// There are no ordering guarantees about these
+pub trait ExtPort {
+	#[must_use]
+	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>;
+	#[must_use]
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>>;
+}
+
+pub struct ExtInit {
+	pub header: api::ExtensionHeader,
+	pub port: Box<dyn ExtPort>,
+}
+impl ExtInit {
+	pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await }
+	pub async fn recv(&self) -> Option<Vec<u8>> { self.port.recv().await }
+}
+impl Deref for ExtInit {
+	type Target = api::ExtensionHeader;
+	fn deref(&self) -> &Self::Target { &self.header }
+}

orchid-base/src/error.rs

@@ -2,16 +2,13 @@ use std::cell::RefCell;
 use std::ffi::OsStr;
 use std::fmt;
 use std::ops::Add;
-use std::rc::Rc;
 use std::sync::Arc;
 
-use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
-use task_local::task_local;
 
 use crate::api;
-use crate::interner::{IStr, es, is};
+use crate::interner::{Interner, Tok};
 use crate::location::Pos;
 
 /// A point of interest in resolving the error, such as the point where
@@ -27,10 +24,10 @@ impl ErrPos {
 	pub fn new(msg: &str, position: Pos) -> Self {
 		Self { message: Some(Arc::new(msg.to_string())), position }
 	}
-	async fn from_api(api: &api::ErrLocation) -> Self {
+	async fn from_api(api: &api::ErrLocation, i: &Interner) -> Self {
 		Self {
 			message: Some(api.message.clone()).filter(|s| !s.is_empty()),
-			position: Pos::from_api(&api.location).await,
+			position: Pos::from_api(&api.location, i).await,
 		}
 	}
 	fn to_api(&self) -> api::ErrLocation {
@@ -54,7 +51,7 @@ impl fmt::Display for ErrPos {
 
 #[derive(Clone, Debug)]
 pub struct OrcErr {
-	pub description: IStr,
+	pub description: Tok<String>,
 	pub message: Arc<String>,
 	pub positions: Vec<ErrPos>,
 }
@@ -66,16 +63,16 @@ impl OrcErr {
 			locations: self.positions.iter().map(ErrPos::to_api).collect(),
 		}
 	}
-	async fn from_api(api: &api::OrcError) -> Self {
+	async fn from_api(api: &api::OrcError, i: &Interner) -> Self {
 		Self {
-			description: es(api.description).await,
+			description: Tok::from_api(api.description, i).await,
 			message: api.message.clone(),
-			positions: join_all(api.locations.iter().map(ErrPos::from_api)).await,
+			positions: join_all(api.locations.iter().map(|e| ErrPos::from_api(e, i))).await,
 		}
 	}
 }
-impl PartialEq<IStr> for OrcErr {
+impl PartialEq<Tok<String>> for OrcErr {
-	fn eq(&self, other: &IStr) -> bool { self.description == *other }
+	fn eq(&self, other: &Tok<String>) -> bool { self.description == *other }
 }
 impl From<OrcErr> for Vec<OrcErr> {
 	fn from(value: OrcErr) -> Self { vec![value] }
@@ -125,10 +122,12 @@ impl OrcErrv {
 		self.0.iter().flat_map(|e| e.positions.iter().cloned())
 	}
 	pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect() }
-	pub async fn from_api<'a>(api: impl IntoIterator<Item = &'a api::OrcError>) -> Self {
+	pub async fn from_api<'a>(
-		Self(join_all(api.into_iter().map(OrcErr::from_api)).await)
+		api: impl IntoIterator<Item = &'a api::OrcError>,
+		i: &Interner,
+	) -> Self {
+		Self(join_all(api.into_iter().map(|e| OrcErr::from_api(e, i))).await)
 	}
-	pub fn iter(&self) -> impl Iterator<Item = OrcErr> + '_ { self.0.iter().cloned() }
 }
 impl From<OrcErr> for OrcErrv {
 	fn from(value: OrcErr) -> Self { Self(vec![value]) }
@@ -192,12 +191,12 @@ macro_rules! join_ok {
 	(@VALUES) => { Ok(()) };
 }
 
-pub fn mk_errv_floating(description: IStr, message: impl AsRef<str>) -> OrcErrv {
+pub fn mk_errv_floating(description: Tok<String>, message: impl AsRef<str>) -> OrcErrv {
 	mk_errv::<Pos>(description, message, [])
 }
 
 pub fn mk_errv<I: Into<ErrPos>>(
-	description: IStr,
+	description: Tok<String>,
 	message: impl AsRef<str>,
 	posv: impl IntoIterator<Item = I>,
 ) -> OrcErrv {
@@ -211,71 +210,45 @@ pub fn mk_errv<I: Into<ErrPos>>(
 
 pub async fn async_io_err<I: Into<ErrPos>>(
 	err: std::io::Error,
+	i: &Interner,
 	posv: impl IntoIterator<Item = I>,
 ) -> OrcErrv {
-	mk_errv(is(&err.kind().to_string()).await, err.to_string(), posv)
+	mk_errv(i.i(&err.kind().to_string()).await, err.to_string(), posv)
 }
 
-pub async fn os_str_to_string<I: Into<ErrPos>>(
+pub async fn os_str_to_string<'a, I: Into<ErrPos>>(
-	str: &OsStr,
+	str: &'a OsStr,
+	i: &Interner,
 	posv: impl IntoIterator<Item = I>,
-) -> OrcRes<&str> {
+) -> OrcRes<&'a str> {
 	match str.to_str() {
 		Some(str) => Ok(str),
 		None => Err(mk_errv(
-			is("Non-unicode string").await,
+			i.i("Non-unicode string").await,
 			format!("{str:?} is not representable as unicode"),
 			posv,
 		)),
 	}
 }
 
-#[derive(Clone, Default)]
+pub struct Reporter {
-struct Reporter {
+	errors: RefCell<Vec<OrcErr>>,
-	errors: Rc<RefCell<Vec<OrcErr>>>,
 }
 
-task_local! {
+impl Reporter {
-	static REPORTER: Reporter;
+	pub fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
+	pub fn new() -> Self { Self { errors: RefCell::new(vec![]) } }
+	pub fn errv(self) -> Option<OrcErrv> { OrcErrv::new(self.errors.into_inner()).ok() }
+	pub fn merge<T>(self, res: OrcRes<T>) -> OrcRes<T> {
+		match (res, self.errv()) {
+			(res, None) => res,
+			(Ok(_), Some(errv)) => Err(errv),
+			(Err(e), Some(errv)) => Err(e + errv),
+		}
+	}
+	pub fn is_empty(&self) -> bool { self.errors.borrow().is_empty() }
 }
 
-/// Run the future with a new reporter, and return all errors reported within.
+impl Default for Reporter {
-///
+	fn default() -> Self { Self::new() }
-/// If your future returns [OrcRes], see [try_with_reporter]
-pub async fn with_reporter<T>(fut: impl Future<Output = T>) -> OrcRes<T> {
-	try_with_reporter(fut.map(Ok)).await
-}
-
-/// Run the future with a new reporter, and return all errors either returned or
-/// reported by it
-///
-/// If your future may report errors but always returns an approximate value,
-/// see [with_reporter]
-pub async fn try_with_reporter<T>(fut: impl Future<Output = OrcRes<T>>) -> OrcRes<T> {
-	let rep = Reporter::default();
-	let res = REPORTER.scope(rep.clone(), fut).await;
-	let errors = rep.errors.take();
-	match (res, &errors[..]) {
-		(Ok(t), []) => Ok(t),
-		(Ok(_), [_, ..]) => Err(OrcErrv::new(errors).unwrap()),
-		(Err(e), _) => Err(e.extended(errors)),
-	}
-}
-
-pub async fn is_erroring() -> bool {
-	(REPORTER.try_with(|r| !r.errors.borrow().is_empty()))
-		.expect("Sidechannel errors must be caught by a reporter")
-}
-
-/// Report an error that is fatal and prevents a correct output, but
-/// still allows the current task to continue and produce an approximate output.
-/// This can be used for
-pub fn report(e: impl Into<OrcErrv>) {
-	let errv = e.into();
-	REPORTER.try_with(|r| r.errors.borrow_mut().extend(errv.clone())).unwrap_or_else(|_| {
-		panic!(
-			"Unhandled error! Sidechannel errors must be caught by an enclosing call to with_reporter.\n\
-			Error: {errv}"
-		)
-	})
 }

orchid-base/src/format.rs

@@ -3,7 +3,6 @@ use std::cmp::Ordering;
 use std::convert::Infallible;
 use std::future::Future;
 use std::iter;
-use std::marker::PhantomData;
 use std::rc::Rc;
 use std::str::FromStr;
 
@@ -12,6 +11,7 @@ use itertools::{Itertools, chain};
 use never::Never;
 use regex::Regex;
 
+use crate::interner::Interner;
 use crate::{api, match_mapping};
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -300,16 +300,16 @@ pub fn take_first(unit: &FmtUnit, bounded: bool) -> String {
 	fill_slots(&first.elements, &unit.subs, 0, bounded)
 }
 
-pub async fn take_first_fmt(v: &(impl Format + ?Sized)) -> String {
+pub async fn take_first_fmt(v: &(impl Format + ?Sized), i: &Interner) -> String {
-	take_first(&v.print(&FmtCtxImpl { _foo: PhantomData }).await, false)
+	take_first(&v.print(&FmtCtxImpl { i }).await, false)
 }
 
-#[derive(Default)]
 pub struct FmtCtxImpl<'a> {
-	_foo: PhantomData<&'a ()>,
+	pub i: &'a Interner,
 }
 
 pub trait FmtCtx {
+	fn i(&self) -> &Interner;
 	// fn print_as(&self, p: &(impl Format + ?Sized)) -> impl Future<Output =
 	// String> where Self: Sized {
 	// 	async {
@@ -319,7 +319,9 @@ pub trait FmtCtx {
 	// 	}
 	// }
 }
-impl FmtCtx for FmtCtxImpl<'_> {}
+impl FmtCtx for FmtCtxImpl<'_> {
+	fn i(&self) -> &Interner { self.i }
+}
 
 pub trait Format {
 	#[must_use]
@@ -330,10 +332,13 @@ impl Format for Never {
 }
 
 /// Format with default strategy. Currently equal to [take_first_fmt]
-pub async fn fmt(v: &(impl Format + ?Sized)) -> String { take_first_fmt(v).await }
+pub async fn fmt(v: &(impl Format + ?Sized), i: &Interner) -> String { take_first_fmt(v, i).await }
 /// Format a sequence with default strategy. Currently equal to [take_first_fmt]
-pub async fn fmt_v<F: Format + ?Sized>(
+pub async fn fmt_v<F: Format + ?Sized, R: Borrow<F>>(
-	v: impl IntoIterator<Item: Borrow<F>>,
+	v: impl IntoIterator<Item = R>,
+	i: &Interner,
 ) -> impl Iterator<Item = String> {
-	join_all(v.into_iter().map(|f| async move { take_first_fmt(f.borrow()).await })).await.into_iter()
+	join_all(v.into_iter().map(|f| async move { take_first_fmt(f.borrow(), i).await }))
+		.await
+		.into_iter()
 }

orchid-base/src/future_debug.rs

@@ -1,170 +0,0 @@
-use std::cell::RefCell;
-use std::fmt::Display;
-use std::pin::pin;
-use std::rc::Rc;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::task::{Context, Poll, Wake, Waker};
-use std::thread::panicking;
-
-use futures::Stream;
-use itertools::Itertools;
-use task_local::task_local;
-
-struct OnPollWaker<F: Fn() + 'static>(Waker, F);
-impl<F: Fn() + 'static> Wake for OnPollWaker<F> {
-	fn wake(self: Arc<Self>) {
-		(self.1)();
-		self.0.wake_by_ref()
-	}
-}
-
-/// Attach a callback to the [Future] protocol for testing and debugging. Note
-/// that this function is safe and simple in order to facilitate debugging
-/// without adding more points of failure, but it's not fast; it performs a heap
-/// allocation on each poll of the returned future.
-pub async fn on_wake<F: Future>(
-	f: F,
-	wake: impl Fn() + Clone + Send + Sync + 'static,
-) -> F::Output {
-	let mut f = pin!(f);
-	futures::future::poll_fn(|cx| {
-		let waker = Arc::new(OnPollWaker(cx.waker().clone(), wake.clone())).into();
-		f.as_mut().poll(&mut Context::from_waker(&waker))
-	})
-	.await
-}
-
-/// Respond to [Future::poll] with a callback. For maximum flexibility and state
-/// control, your callback receives the actual poll job as a callback function.
-/// Failure to call this function will result in an immediate panic.
-pub async fn wrap_poll<Fut: Future>(
-	f: Fut,
-	mut cb: impl FnMut(Box<dyn FnOnce() -> bool + '_>),
-) -> Fut::Output {
-	let mut f = pin!(f);
-	futures::future::poll_fn(|cx| {
-		let poll = RefCell::new(None);
-		cb(Box::new(|| {
-			let poll1 = f.as_mut().poll(cx);
-			let ret = poll1.is_ready();
-			*poll.borrow_mut() = Some(poll1);
-			ret
-		}));
-		poll.into_inner().expect("Callback to on_poll failed to call its argument")
-	})
-	.await
-}
-
-pub fn wrap_poll_next<'a, S: Stream + 'a>(
-	s: S,
-	mut cb: impl FnMut(Box<dyn FnOnce() -> bool + '_>) + 'a,
-) -> impl Stream<Item = S::Item> + 'a {
-	let mut s = Box::pin(s);
-	futures::stream::poll_fn(move |cx| {
-		let poll = RefCell::new(None);
-		cb(Box::new(|| {
-			let poll1 = s.as_mut().poll_next(cx);
-			let ret = poll1.is_ready();
-			*poll.borrow_mut() = Some(poll1);
-			ret
-		}));
-		poll.into_inner().expect("Callback to on_poll failed to call its argument")
-	})
-}
-
-pub fn on_stream_wake<'a, S: Stream + 'a>(
-	s: S,
-	wake: impl Fn() + Clone + Send + Sync + 'static,
-) -> impl Stream<Item = S::Item> {
-	let mut s = Box::pin(s);
-	futures::stream::poll_fn(move |cx| {
-		let waker = Arc::new(OnPollWaker(cx.waker().clone(), wake.clone())).into();
-		s.as_mut().poll_next(&mut Context::from_waker(&waker))
-	})
-}
-
-task_local! {
-	static LABEL_STATE: Vec<Rc<String>>
-}
-
-pub async fn with_label<Fut: Future>(label: &str, f: Fut) -> Fut::Output {
-	let mut new_lbl = LABEL_STATE.try_with(|lbl| lbl.clone()).unwrap_or_default();
-	new_lbl.push(Rc::new(label.to_string()));
-	LABEL_STATE.scope(new_lbl, f).await
-}
-
-pub fn label() -> impl Display + Clone + Send + Sync + 'static {
-	LABEL_STATE.try_with(|lbl| lbl.iter().join("/")).unwrap_or("".to_string())
-}
-
-pub struct Label;
-impl Display for Label {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", label()) }
-}
-
-pub async fn eprint_events<Fut: Future>(note: &str, f: Fut) -> Fut::Output {
-	let label = label();
-	let note1 = note.to_string();
-	on_wake(
-		wrap_poll(f, |cb| {
-			eprintln!("{Label} polling {note}");
-			eprintln!("{Label} polled {note} (ready? {})", cb())
-		}),
-		move || eprintln!("{label} woke {note1}"),
-	)
-	.await
-}
-
-pub fn eprint_stream_events<'a, S: Stream + 'a>(
-	note: &'a str,
-	s: S,
-) -> impl Stream<Item = S::Item> + 'a {
-	let label = label();
-	let note1 = note.to_string();
-	on_stream_wake(
-		wrap_poll_next(s, move |cb| {
-			eprintln!("{Label} polling {note}");
-			eprintln!("{Label} polled {note} (ready? {})", cb())
-		}),
-		move || eprintln!("{label} woke {note1}"),
-	)
-}
-
-struct SpinWaker(AtomicBool);
-impl Wake for SpinWaker {
-	fn wake(self: Arc<Self>) { self.0.store(true, Ordering::Relaxed); }
-}
-
-/// A dumb executor that keeps synchronously re-running the future as long as it
-/// keeps synchronously waking itself.
-///
-/// # Panics
-///
-/// If the future doesn't wake itself and doesn't settle. This is useful for
-/// deterministic tests that don't contain side effects or threading.
-pub fn spin_on<Fut: Future>(f: Fut) -> Fut::Output {
-	let repeat = Arc::new(SpinWaker(AtomicBool::new(false)));
-	let mut f = pin!(f);
-	let waker = repeat.clone().into();
-	let mut cx = Context::from_waker(&waker);
-	loop {
-		match f.as_mut().poll(&mut cx) {
-			Poll::Ready(t) => break t,
-			Poll::Pending if repeat.0.swap(false, Ordering::Relaxed) => (),
-			Poll::Pending => panic!("The future did not exit and did not call its waker."),
-		}
-	}
-}
-
-/// Create an object that will panic if dropped. [PanicOnDrop::defuse] must be
-/// called once the particular constraint preventing a drop has passed
-pub fn assert_no_drop(msg: &'static str) -> PanicOnDrop { PanicOnDrop(true, msg) }
-
-pub struct PanicOnDrop(bool, &'static str);
-impl PanicOnDrop {
-	pub fn defuse(mut self) { self.0 = false; }
-}
-impl Drop for PanicOnDrop {
-	fn drop(&mut self) { assert!(panicking() || !self.0, "{}", self.1) }
-}

orchid-base/src/interner.rs

@@ -1,382 +1,310 @@
-use std::fmt::{Debug, Display};
+use std::borrow::Borrow;
 use std::future::Future;
-use std::hash::Hash;
+use std::hash::BuildHasher as _;
+use std::num::NonZeroU64;
 use std::ops::Deref;
 use std::rc::Rc;
+use std::sync::atomic;
 use std::{fmt, hash};
 
-use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
-use task_local::task_local;
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools as _;
+use orchid_api_traits::Request;
 
 use crate::api;
+use crate::reqnot::{DynRequester, Requester};
 
-pub trait IStrHandle: AsRef<str> {
+/// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
-	fn rc(&self) -> Rc<String>;
+/// a minimal example
-}
+#[derive(Clone)]
-pub trait IStrvHandle: AsRef<[IStr]> {
+struct ForceSized<T>(T);
-	fn rc(&self) -> Rc<Vec<IStr>>;
-}
 
 #[derive(Clone)]
-pub struct IStr(pub api::TStr, pub Rc<dyn IStrHandle>);
+pub struct Tok<T: Interned> {
-impl IStr {
+	data: Rc<T>,
-	/// Obtain a unique ID for this interned data.
+	marker: ForceSized<T::Marker>,
-	///
-	/// NOTICE: the ID is guaranteed to be the same for any interned instance of
-	/// the same value only as long as at least one instance exists. If a value is
-	/// no longer interned, the interner is free to forget about it.
-	pub fn to_api(&self) -> api::TStr { self.0 }
-	pub fn rc(&self) -> Rc<String> { self.1.rc() }
 }
-impl Deref for IStr {
+impl<T: Interned> Tok<T> {
-	type Target = str;
+	pub fn new(data: Rc<T>, marker: T::Marker) -> Self { Self { data, marker: ForceSized(marker) } }
-	fn deref(&self) -> &Self::Target { self.1.as_ref().as_ref() }
+	pub fn to_api(&self) -> T::Marker { self.marker.0 }
+	pub async fn from_api<M>(marker: M, i: &Interner) -> Self
+	where M: InternMarker<Interned = T> {
+		i.ex(marker).await
 	}
-impl Eq for IStr {}
+	pub fn rc(&self) -> Rc<T> { self.data.clone() }
-impl PartialEq for IStr {
-	fn eq(&self, other: &Self) -> bool { self.0 == other.0 }
 }
-impl Hash for IStr {
+impl<T: Interned> Deref for Tok<T> {
-	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.0.hash(state) }
+	type Target = T;
+
+	fn deref(&self) -> &Self::Target { self.data.as_ref() }
 }
-impl Display for IStr {
+impl<T: Interned> Ord for Tok<T> {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.deref()) }
+	fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.to_api().cmp(&other.to_api()) }
 }
-impl Debug for IStr {
+impl<T: Interned> PartialOrd for Tok<T> {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "IStr({self}") }
+	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
 }
-#[derive(Clone)]
+impl<T: Interned> Eq for Tok<T> {}
-pub struct IStrv(pub api::TStrv, pub Rc<dyn IStrvHandle>);
+impl<T: Interned> PartialEq for Tok<T> {
-impl IStrv {
+	fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
-	/// Obtain a unique ID for this interned data.
-	///
-	/// NOTICE: the ID is guaranteed to be the same for any interned instance of
-	/// the same value only as long as at least one instance exists. If a value is
-	/// no longer interned, the interner is free to forget about it.
-	pub fn to_api(&self) -> api::TStrv { self.0 }
-	pub fn rc(&self) -> Rc<Vec<IStr>> { self.1.rc() }
 }
-impl Deref for IStrv {
+impl<T: Interned> hash::Hash for Tok<T> {
-	type Target = [IStr];
+	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.to_api().hash(state) }
-	fn deref(&self) -> &Self::Target { self.1.as_ref().as_ref() }
 }
-impl Eq for IStrv {}
+impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
-impl PartialEq for IStrv {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-	fn eq(&self, other: &Self) -> bool { self.0 == other.0 }
+		write!(f, "{}", &*self.data)
 	}
-impl Hash for IStrv {
-	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.0.0.hash(state) }
 }
-impl Display for IStrv {
+impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let mut iter = self.deref().iter();
+		write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
-		match iter.next() {
-			None => return Ok(()),
-			Some(s) => write!(f, "{s}")?,
 	}
-		for s in iter {
-			write!(f, "::{s}")?
-		}
-		Ok(())
-	}
-}
-impl Debug for IStrv {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "IStrv({self})") }
 }
 
-pub trait InternerSrv {
+pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug + Internable<Interned = Self> {
-	fn is<'a>(&'a self, v: &'a str) -> LocalBoxFuture<'a, IStr>;
+	type Marker: InternMarker<Interned = Self> + Sized;
-	fn es(&self, t: api::TStr) -> LocalBoxFuture<'_, IStr>;
+	fn intern(
-	fn iv<'a>(&'a self, v: &'a [IStr]) -> LocalBoxFuture<'a, IStrv>;
+		self: Rc<Self>,
-	fn ev(&self, t: api::TStrv) -> LocalBoxFuture<'_, IStrv>;
+		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+	) -> impl Future<Output = Self::Marker>;
+	fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
 }
 
-task_local! {
+pub trait Internable: fmt::Debug {
-	static INTERNER: Rc<dyn InternerSrv>;
+	type Interned: Interned;
+	fn get_owned(&self) -> Rc<Self::Interned>;
 }
 
-pub async fn with_interner<F: Future>(val: Rc<dyn InternerSrv>, fut: F) -> F::Output {
+pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash + Sized {
-	INTERNER.scope(val, fut).await
+	type Interned: Interned<Marker = Self>;
+	/// Only called on replicas
+	fn resolve(self, i: &Interner) -> impl Future<Output = Tok<Self::Interned>>;
+	fn get_id(self) -> NonZeroU64;
+	fn from_id(id: NonZeroU64) -> Self;
 }
 
-fn get_interner() -> Rc<dyn InternerSrv> {
+impl Interned for String {
-	INTERNER.try_with(|i| i.clone()).expect("Interner not initialized")
+	type Marker = api::TStr;
+	async fn intern(
+		self: Rc<Self>,
+		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+	) -> Self::Marker {
+		req.request(api::InternStr(self.to_string())).await
+	}
+	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.strings }
+}
+impl InternMarker for api::TStr {
+	type Interned = String;
+	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
+		Tok::new(Rc::new(i.0.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
+	}
+	fn get_id(self) -> NonZeroU64 { self.0 }
+	fn from_id(id: NonZeroU64) -> Self { Self(id) }
+}
+impl Internable for str {
+	type Interned = String;
+	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
+}
+impl Internable for String {
+	type Interned = String;
+	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
 }
 
-pub async fn is(v: &str) -> IStr { get_interner().is(v).await }
+impl Interned for Vec<Tok<String>> {
-pub async fn iv(v: &[IStr]) -> IStrv { get_interner().iv(v).await }
+	type Marker = api::TStrv;
-pub async fn es(v: api::TStr) -> IStr { get_interner().es(v).await }
+	async fn intern(
-pub async fn ev(v: api::TStrv) -> IStrv { get_interner().ev(v).await }
+		self: Rc<Self>,
+		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+	) -> Self::Marker {
+		req.request(api::InternStrv(self.iter().map(|t| t.to_api()).collect())).await
+	}
+	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
+}
+impl InternMarker for api::TStrv {
+	type Interned = Vec<Tok<String>>;
+	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
+		let rep = i.0.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
+		let data = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
+		Tok::new(Rc::new(data), self)
+	}
+	fn get_id(self) -> NonZeroU64 { self.0 }
+	fn from_id(id: NonZeroU64) -> Self { Self(id) }
+}
+impl Internable for [Tok<String>] {
+	type Interned = Vec<Tok<String>>;
+	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+}
+impl<const N: usize> Internable for [Tok<String>; N] {
+	type Interned = Vec<Tok<String>>;
+	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+}
+impl Internable for Vec<Tok<String>> {
+	type Interned = Vec<Tok<String>>;
+	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+}
+// impl Internable for Vec<api::TStr> {
+// 	type Interned = Vec<Tok<String>>;
+// 	fn get_owned(&self) -> Arc<Self::Interned> {
+// 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
+// 	}
+// }
+// impl Internable for [api::TStr] {
+// 	type Interned = Vec<Tok<String>>;
+// 	fn get_owned(&self) -> Arc<Self::Interned> {
+// 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
+// 	}
+// }
 
-pub mod local_interner {
+/// The number of references held to any token by the interner.
-	use std::borrow::Borrow;
+const BASE_RC: usize = 3;
-	use std::cell::RefCell;
-	use std::fmt::Debug;
-	use std::future;
-	use std::hash::{BuildHasher, Hash};
-	use std::num::NonZeroU64;
-	use std::rc::{Rc, Weak};
 
-	use futures::future::LocalBoxFuture;
+#[test]
-	use hashbrown::hash_table::{Entry, OccupiedEntry, VacantEntry};
+fn base_rc_correct() {
-	use hashbrown::{DefaultHashBuilder, HashTable};
+	let tok = Tok::new(Rc::new("foo".to_string()), api::TStr(1.try_into().unwrap()));
-	use orchid_api_traits::Coding;
+	let mut bimap = Bimap::default();
+	bimap.insert(tok.clone());
+	assert_eq!(Rc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the current instance");
+}
 
-	use super::{IStr, IStrHandle, IStrv, IStrvHandle, InternerSrv};
+pub struct Bimap<T: Interned> {
+	intern: HashMap<Rc<T>, Tok<T>>,
+	by_id: HashMap<T::Marker, Tok<T>>,
+}
+impl<T: Interned> Bimap<T> {
+	pub fn insert(&mut self, token: Tok<T>) {
+		self.intern.insert(token.data.clone(), token.clone());
+		self.by_id.insert(token.to_api(), token);
+	}
+
+	pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> { self.by_id.get(&marker).cloned() }
+
+	pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Tok<T>>
+	where T: Borrow<Q> {
+		(self.intern.raw_entry())
+			.from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
+			.map(|p| p.1.clone())
+	}
+
+	pub fn sweep_replica(&mut self) -> Vec<T::Marker> {
+		(self.intern)
+			.extract_if(|k, _| Rc::strong_count(k) == BASE_RC)
+			.map(|(_, v)| {
+				self.by_id.remove(&v.to_api());
+				v.to_api()
+			})
+			.collect()
+	}
+
+	pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
+		self.intern.retain(|k, v| BASE_RC < Rc::strong_count(k) || retained.contains(&v.to_api()))
+	}
+}
+
+impl<T: Interned> Default for Bimap<T> {
+	fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new() } }
+}
+
+pub trait UpComm {
+	fn up<R: Request>(&self, req: R) -> R::Response;
+}
+
+#[derive(Default)]
+pub struct TypedInterners {
+	strings: Bimap<String>,
+	vecs: Bimap<Vec<Tok<String>>>,
+}
+
+#[derive(Default)]
+pub struct InternerData {
+	interners: Mutex<TypedInterners>,
+	master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
+}
+#[derive(Clone, Default)]
+pub struct Interner(Rc<InternerData>);
+impl Interner {
+	pub fn new_master() -> Self { Self::default() }
+	pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) -> Self {
+		Self(Rc::new(InternerData { master: Some(Box::new(req)), interners: Mutex::default() }))
+	}
+	/// Intern some data; query its identifier if not known locally
+	pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
+		let data = t.get_owned();
+		let mut g = self.0.interners.lock().await;
+		let typed = T::bimap(&mut g);
+		if let Some(tok) = typed.by_value(&data) {
+			return tok;
+		}
+		let marker = match &self.0.master {
+			Some(c) => data.clone().intern(&**c).await,
+			None =>
+				T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
+		};
+		let tok = Tok::new(data, marker);
+		T::bimap(&mut g).insert(tok.clone());
+		tok
+	}
+	/// Extern an identifier; query the data it represents if not known locally
+	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
+		if let Some(tok) = M::Interned::bimap(&mut *self.0.interners.lock().await).by_marker(marker) {
+			return tok;
+		}
+		assert!(self.0.master.is_some(), "ID not in local interner and this is master");
+		let token = marker.resolve(self).await;
+		M::Interned::bimap(&mut *self.0.interners.lock().await).insert(token.clone());
+		token
+	}
+	pub async fn sweep_replica(&self) -> api::Retained {
+		assert!(self.0.master.is_some(), "Not a replica");
+		let mut g = self.0.interners.lock().await;
+		api::Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
+	}
+	pub async fn sweep_master(&self, retained: api::Retained) {
+		assert!(self.0.master.is_none(), "Not master");
+		let mut g = self.0.interners.lock().await;
+		g.strings.sweep_master(retained.strings.into_iter().collect());
+		g.vecs.sweep_master(retained.vecs.into_iter().collect());
+	}
+}
+impl fmt::Debug for Interner {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "Interner{{ replica: {} }}", self.0.master.is_none())
+	}
+}
+
+static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
+
+pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
+	into.strings = into.strings.iter().chain(&from.strings).copied().unique().collect();
+	into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
+}
+
+#[cfg(test)]
+mod test {
+	use std::num::NonZero;
+	use std::pin::Pin;
+
+	use orchid_api_traits::{Decode, enc_vec};
+	use test_executors::spin_on;
+
+	use super::*;
 	use crate::api;
 
-	/// Associated types and methods for parallel concepts between scalar and
+	#[test]
-	/// vector interning
+	fn test_i() {
-	pub trait InternableCard: 'static + Sized + Default + Debug {
+		let i = Interner::new_master();
-		/// API representation of an interner key
+		let _: Tok<String> = spin_on(i.i("foo"));
-		type Token: Clone + Copy + Debug + Hash + Eq + PartialOrd + Ord + Coding + 'static;
+		let _: Tok<Vec<Tok<String>>> = spin_on(i.i(&[spin_on(i.i("bar")), spin_on(i.i("baz"))]));
-		/// Owned version of interned value physically held by `'static` interner
-		/// and token
-		type Data: 'static + Borrow<Self::Borrow> + Eq + Hash + Debug;
-		/// Borrowed version of interned value placed in intern queries to avoid a
-		/// copy
-		type Borrow: ToOwned<Owned = Self::Data> + ?Sized + Eq + Hash + Debug;
-		/// Smart object handed out by the interner for storage and comparison in
-		/// third party code. [IStr] or [IStrv]
-		type Interned: Clone + Debug;
-		/// Create smart object from token for fast comparison and a handle for
-		/// everything else incl. virtual drop
-		fn new_interned(token: Self::Token, handle: Rc<Handle<Self>>) -> Self::Interned;
 	}
 
-	#[derive(Default, Debug)]
+	#[test]
-	pub struct StrBranch;
+	fn test_coding() {
-	impl InternableCard for StrBranch {
+		spin_on(async {
-		type Data = String;
+			let coded = api::TStr(NonZero::new(3u64).unwrap());
-		type Token = api::TStr;
+			let mut enc = &enc_vec(&coded).await[..];
-		type Borrow = str;
+			api::TStr::decode(Pin::new(&mut enc)).await;
-		type Interned = IStr;
+			assert_eq!(enc, [], "Did not consume all of {enc:?}")
-		fn new_interned(t: Self::Token, h: Rc<Handle<Self>>) -> Self::Interned { IStr(t, h) }
+		})
-	}
-
-	#[derive(Default, Debug)]
-	pub struct StrvBranch;
-	impl InternableCard for StrvBranch {
-		type Data = Vec<IStr>;
-		type Token = api::TStrv;
-		type Borrow = [IStr];
-		type Interned = IStrv;
-		fn new_interned(t: Self::Token, h: Rc<Handle<Self>>) -> Self::Interned { IStrv(t, h) }
-	}
-
-	/// Pairs interned data with its internment key
-	#[derive(Debug)]
-	struct Data<B: InternableCard> {
-		token: B::Token,
-		data: Rc<B::Data>,
-	}
-	impl<B: InternableCard> Clone for Data<B> {
-		fn clone(&self) -> Self { Self { token: self.token, data: self.data.clone() } }
-	}
-
-	/// Implementor for the trait objects held by [IStr] and [IStrv]
-	pub struct Handle<B: InternableCard> {
-		data: Data<B>,
-		parent: Weak<RefCell<IntData<B>>>,
-	}
-	impl IStrHandle for Handle<StrBranch> {
-		fn rc(&self) -> Rc<String> { self.data.data.clone() }
-	}
-	impl AsRef<str> for Handle<StrBranch> {
-		fn as_ref(&self) -> &str { self.data.data.as_ref().as_ref() }
-	}
-	impl IStrvHandle for Handle<StrvBranch> {
-		fn rc(&self) -> Rc<Vec<IStr>> { self.data.data.clone() }
-	}
-	impl AsRef<[IStr]> for Handle<StrvBranch> {
-		fn as_ref(&self) -> &[IStr] { self.data.data.as_ref().as_ref() }
-	}
-	impl<B: InternableCard> Drop for Handle<B> {
-		fn drop(&mut self) {
-			let Some(parent) = self.parent.upgrade() else { return };
-			if let Entry::Occupied(ent) =
-				parent.borrow_mut().entry_by_data(self.data.data.as_ref().borrow())
-			{
-				ent.remove();
-			}
-			if let Entry::Occupied(ent) = parent.borrow_mut().entry_by_tok(self.data.token) {
-				ent.remove();
 	}
 }
-	}
-
-	/// Information retained about an interned token indexed both by key and
-	/// value.
-	struct Rec<B: InternableCard> {
-		/// This reference is weak, but the [Drop] handler of [Handle] removes all
-		/// [Rec]s from the interner so it is guaranteed to be live.
-		handle: Weak<Handle<B>>,
-		/// Keys for indexing from either table
-		data: Data<B>,
-	}
-
-	/// Read data from an occupied entry in an interner. The equivalent insert
-	/// command is [insert]
-	fn read<B: InternableCard>(entry: OccupiedEntry<'_, Rec<B>>) -> B::Interned {
-		let hand = entry.get().handle.upgrade().expect("Found entry but handle already dropped");
-		B::new_interned(entry.get().data.token, hand)
-	}
-
-	/// Insert some data into an entry borrowed from this same interner.
-	/// The equivalent read command is [read]
-	fn insert<B: InternableCard>(entry: VacantEntry<'_, Rec<B>>, handle: Rc<Handle<B>>) {
-		entry.insert(Rec { data: handle.data.clone(), handle: Rc::downgrade(&handle) });
-	}
-
-	#[derive(Default)]
-	struct IntData<B: InternableCard> {
-		by_tok: HashTable<Rec<B>>,
-		by_data: HashTable<Rec<B>>,
-		hasher: DefaultHashBuilder,
-	}
-	impl<B: InternableCard> IntData<B> {
-		fn entry_by_data(&mut self, query: &B::Borrow) -> Entry<'_, Rec<B>> {
-			self.by_data.entry(
-				self.hasher.hash_one(query),
-				|rec| rec.data.data.as_ref().borrow() == query,
-				|rec| self.hasher.hash_one(rec.data.data.as_ref().borrow()),
-			)
-		}
-		fn entry_by_tok(&mut self, token: B::Token) -> Entry<'_, Rec<B>> {
-			self.by_tok.entry(
-				self.hasher.hash_one(token),
-				|rec| rec.data.token == token,
-				|rec| self.hasher.hash_one(rec.data.token),
-			)
-		}
-	}
-
-	/// Failing intern command that can be recovered if the value is found
-	/// elsewhere
-	pub struct InternError<'a, B: InternableCard> {
-		int: &'a Int<B>,
-		query: &'a B::Borrow,
-	}
-	impl<B: InternableCard> InternError<'_, B> {
-		/// If a racing write populates the entry, the continuation returns that
-		/// value and discards its argument
-		pub fn set_if_empty(self, token: B::Token) -> B::Interned {
-			let mut int_data = self.int.0.borrow_mut();
-			match int_data.entry_by_data(self.query) {
-				Entry::Occupied(ent) => read(ent),
-				Entry::Vacant(ent) => {
-					let hand = self.int.mk_handle(Data { token, data: Rc::new(self.query.to_owned()) });
-					insert(ent, hand.clone());
-					let Entry::Vacant(other_ent) = int_data.entry_by_tok(token) else {
-						panic!("Data and key tables out of sync")
-					};
-					insert(other_ent, hand.clone());
-					B::new_interned(token, hand)
-				},
-			}
-		}
-	}
-	impl<B: InternableCard> Debug for InternError<'_, B> {
-		fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-			f.debug_tuple("InternEntry").field(&self.query).finish()
-		}
-	}
-
-	/// Failing extern command that can be recovered if the value is found
-	/// elsewhere
-	pub struct ExternError<'a, B: InternableCard> {
-		int: &'a Int<B>,
-		token: B::Token,
-	}
-	impl<B: InternableCard> ExternError<'_, B> {
-		/// If a racing write populates the entry, the continuation returns that
-		/// value and discards its argument
-		pub fn set_if_empty(&self, data: Rc<B::Data>) -> B::Interned {
-			let mut int_data = self.int.0.borrow_mut();
-			match int_data.entry_by_tok(self.token) {
-				Entry::Occupied(ent) => read(ent),
-				Entry::Vacant(ent) => {
-					let hand = self.int.mk_handle(Data { token: self.token, data: data.clone() });
-					insert(ent, hand.clone());
-					let Entry::Vacant(other_ent) = int_data.entry_by_data(data.as_ref().borrow()) else {
-						panic!("Data and key tables out of sync")
-					};
-					insert(other_ent, hand.clone());
-					B::new_interned(self.token, hand)
-				},
-			}
-		}
-	}
-	impl<B: InternableCard> Debug for ExternError<'_, B> {
-		fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-			f.debug_tuple("ExternEntry").field(&self.token).finish()
-		}
-	}
-
-	#[derive(Default)]
-	pub struct Int<B: InternableCard>(Rc<RefCell<IntData<B>>>);
-	impl<B: InternableCard> Int<B> {
-		fn mk_handle(&self, data: Data<B>) -> Rc<Handle<B>> {
-			Rc::new(Handle { data: data.clone(), parent: Rc::downgrade(&self.0.clone()) })
-		}
-
-		/// Look up by value, or yield to figure out its ID from elsewhere
-		pub fn i<'a>(&'a self, query: &'a B::Borrow) -> Result<B::Interned, InternError<'a, B>> {
-			if let Entry::Occupied(val) = self.0.borrow_mut().entry_by_data(query) {
-				return Ok(read(val));
-			}
-			Err(InternError { int: self, query })
-		}
-
-		/// Look up by key or yield to figure out its value from elsewhere
-		pub fn e(&self, token: B::Token) -> Result<B::Interned, ExternError<'_, B>> {
-			if let Entry::Occupied(ent) = self.0.borrow_mut().entry_by_tok(token) {
-				return Ok(read(ent));
-			}
-			Err(ExternError { int: self, token })
-		}
-	}
-
-	thread_local! {
-		static NEXT_ID: RefCell<u64> = 0.into();
-	}
-
-	fn with_new_id<T>(fun: impl FnOnce(NonZeroU64) -> T) -> T {
-		fun(
-			NonZeroU64::new(NEXT_ID.with_borrow_mut(|id| {
-				*id += 1;
-				*id
-			}))
-			.unwrap(),
-		)
-	}
-
-	#[derive(Default)]
-	struct LocalInterner {
-		str: Int<StrBranch>,
-		strv: Int<StrvBranch>,
-	}
-	impl InternerSrv for LocalInterner {
-		fn is<'a>(&'a self, v: &'a str) -> LocalBoxFuture<'a, IStr> {
-			match self.str.i(v) {
-				Ok(int) => Box::pin(future::ready(int)),
-				Err(e) => with_new_id(|id| Box::pin(future::ready(e.set_if_empty(api::TStr(id))))),
-			}
-		}
-		fn es(&self, t: api::TStr) -> LocalBoxFuture<'_, IStr> {
-			Box::pin(future::ready(self.str.e(t).expect("Unrecognized token cannot be externed")))
-		}
-		fn iv<'a>(&'a self, v: &'a [IStr]) -> LocalBoxFuture<'a, IStrv> {
-			match self.strv.i(v) {
-				Ok(int) => Box::pin(future::ready(int)),
-				Err(e) => with_new_id(|id| Box::pin(future::ready(e.set_if_empty(api::TStrv(id))))),
-			}
-		}
-		fn ev(&self, t: orchid_api::TStrv) -> LocalBoxFuture<'_, IStrv> {
-			Box::pin(future::ready(self.strv.e(t).expect("Unrecognized token cannot be externed")))
-		}
-	}
-
-	/// Creates a basic thread-local interner for testing and root role.
-	pub fn local_interner() -> Rc<dyn InternerSrv> { Rc::<LocalInterner>::default() }
-}

orchid-base/src/lib.rs

@@ -1,21 +1,19 @@
 pub use async_once_cell;
 use orchid_api as api;
 
-pub mod binary;
 pub mod box_cow;
 pub mod boxed_iter;
+pub mod builtin;
 pub mod char_filter;
 pub mod clone;
 pub mod combine;
 pub mod error;
 pub mod event;
 pub mod format;
-pub mod future_debug;
 pub mod id_store;
 pub mod interner;
 pub mod iter_utils;
 pub mod join;
-mod localset;
 pub mod location;
 pub mod logging;
 mod match_mapping;
@@ -27,7 +25,6 @@ pub mod pure_seq;
 pub mod reqnot;
 pub mod sequence;
 pub mod side;
-pub mod stash;
+mod tl_cache;
-pub mod tl_cache;
 pub mod tokens;
 pub mod tree;

orchid-base/src/localset.rs

@@ -1,48 +0,0 @@
-use std::collections::VecDeque;
-use std::pin::Pin;
-use std::task::Poll;
-
-use futures::StreamExt;
-use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender, unbounded};
-use futures::future::LocalBoxFuture;
-
-pub struct LocalSet<'a, E> {
-	receiver: UnboundedReceiver<LocalBoxFuture<'a, Result<(), E>>>,
-	pending: VecDeque<LocalBoxFuture<'a, Result<(), E>>>,
-}
-impl<'a, E> LocalSet<'a, E> {
-	pub fn new() -> (UnboundedSender<LocalBoxFuture<'a, Result<(), E>>>, Self) {
-		let (sender, receiver) = unbounded();
-		(sender, Self { receiver, pending: VecDeque::new() })
-	}
-}
-impl<E> Future for LocalSet<'_, E> {
-	type Output = Result<(), E>;
-	fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
-		let this = self.get_mut();
-		let mut any_pending = false;
-		loop {
-			match this.receiver.poll_next_unpin(cx) {
-				Poll::Ready(Some(fut)) => this.pending.push_back(fut),
-				Poll::Ready(None) => break,
-				Poll::Pending => {
-					any_pending = true;
-					break;
-				},
-			}
-		}
-		let count = this.pending.len();
-		for _ in 0..count {
-			let mut req = this.pending.pop_front().unwrap();
-			match req.as_mut().poll(cx) {
-				Poll::Ready(Ok(())) => (),
-				Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
-				Poll::Pending => {
-					any_pending = true;
-					this.pending.push_back(req)
-				},
-			}
-		}
-		if any_pending { Poll::Pending } else { Poll::Ready(Ok(())) }
-	}
-}

orchid-base/src/location.rs

@@ -8,12 +8,12 @@ use futures::future::join_all;
 use trait_set::trait_set;
 
 use crate::error::ErrPos;
-use crate::interner::{IStr, es, is};
+use crate::interner::{Interner, Tok};
 use crate::name::Sym;
 use crate::{api, match_mapping, sym};
 
 trait_set! {
-	pub trait GetSrc = FnMut(&Sym) -> IStr;
+	pub trait GetSrc = FnMut(&Sym) -> Tok<String>;
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -37,13 +37,13 @@ impl Pos {
 			other => format!("{other:?}"),
 		}
 	}
-	pub async fn from_api(api: &api::Location) -> Self {
+	pub async fn from_api(api: &api::Location, i: &Interner) -> Self {
 		match_mapping!(api, api::Location => Pos {
 			None, Inherit, SlotTarget,
-			Gen(cgi => CodeGenInfo::from_api(cgi).await),
+			Gen(cgi => CodeGenInfo::from_api(cgi, i).await),
-			Multi(v => join_all(v.iter().map(Pos::from_api)).await)
+			Multi(v => join_all(v.iter().map(|l| Pos::from_api(l, i))).await)
 		} {
-			api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr).await)
+			api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr, i).await)
 		})
 	}
 	pub fn to_api(&self) -> api::Location {
@@ -108,7 +108,7 @@ impl SrcRange {
 	}
 	/// Create a dud [SourceRange] for testing. Its value is unspecified and
 	/// volatile.
-	pub async fn mock() -> Self { Self { range: 0..1, path: sym!(test) } }
+	pub async fn mock(i: &Interner) -> Self { Self { range: 0..1, path: sym!(test; i) } }
 	/// Path the source text was loaded from
 	pub fn path(&self) -> Sym { self.path.clone() }
 	/// Byte range
@@ -133,8 +133,8 @@ impl SrcRange {
 		}
 	}
 	pub fn zw(path: Sym, pos: u32) -> Self { Self { path, range: pos..pos } }
-	pub async fn from_api(api: &api::SourceRange) -> Self {
+	pub async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
-		Self { path: Sym::from_api(api.path).await, range: api.range.clone() }
+		Self { path: Sym::from_api(api.path, i).await, range: api.range.clone() }
 	}
 	pub fn to_api(&self) -> api::SourceRange {
 		api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
@@ -162,19 +162,24 @@ pub struct CodeGenInfo {
 	/// formatted like a Rust namespace
 	pub generator: Sym,
 	/// Unformatted user message with relevant circumstances and parameters
-	pub details: IStr,
+	pub details: Tok<String>,
 }
 impl CodeGenInfo {
 	/// A codegen marker with no user message and parameters
-	pub async fn new_short(generator: Sym) -> Self { Self { generator, details: is("").await } }
+	pub async fn new_short(generator: Sym, i: &Interner) -> Self {
+		Self { generator, details: i.i("").await }
+	}
 	/// A codegen marker with a user message or parameters
-	pub async fn new_details(generator: Sym, details: impl AsRef<str>) -> Self {
+	pub async fn new_details(generator: Sym, details: impl AsRef<str>, i: &Interner) -> Self {
-		Self { generator, details: is(details.as_ref()).await }
+		Self { generator, details: i.i(details.as_ref()).await }
 	}
 	/// Syntactic location
 	pub fn pos(&self) -> Pos { Pos::Gen(self.clone()) }
-	pub async fn from_api(api: &api::CodeGenInfo) -> Self {
+	pub async fn from_api(api: &api::CodeGenInfo, i: &Interner) -> Self {
-		Self { generator: Sym::from_api(api.generator).await, details: es(api.details).await }
+		Self {
+			generator: Sym::from_api(api.generator, i).await,
+			details: Tok::from_api(api.details, i).await,
+		}
 	}
 	pub fn to_api(&self) -> api::CodeGenInfo {
 		api::CodeGenInfo { generator: self.generator.to_api(), details: self.details.to_api() }

orchid-base/src/logging.rs

@@ -1,77 +1,35 @@
-use std::any::Any;
-use std::cell::RefCell;
 use std::fmt::Arguments;
-use std::io::Write;
+use std::fs::File;
-use std::rc::Rc;
+use std::io::{Write, stderr};
 
-use futures::future::LocalBoxFuture;
+pub use api::LogStrategy;
-use task_local::task_local;
+use itertools::Itertools;
 
 use crate::api;
 
-task_local! {
-	static DEFAULT_WRITER: RefCell<Box<dyn Write>>
-}
-
-/// Set the stream used for [api::LogStrategy::Default]. If not set,
-/// [std::io::stderr] will be used.
-pub async fn with_default_stream<F: Future>(stderr: impl Write + 'static, fut: F) -> F::Output {
-	DEFAULT_WRITER.scope(RefCell::new(Box::new(stderr)), fut).await
-}
-
-pub trait LogWriter {
-	fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()>;
-}
-
-pub trait Logger: Any {
-	fn writer(&self, category: &str) -> Rc<dyn LogWriter>;
-	fn strat(&self, category: &str) -> api::LogStrategy;
-	fn is_active(&self, category: &str) -> bool {
-		!matches!(self.strat(category), api::LogStrategy::Discard)
-	}
-}
-
-task_local! {
-	static LOGGER: Rc<dyn Logger>;
-}
-
-pub async fn with_logger<F: Future>(logger: impl Logger + 'static, fut: F) -> F::Output {
-	LOGGER.scope(Rc::new(logger), fut).await
-}
-
-pub fn log(category: &str) -> Rc<dyn LogWriter> {
-	LOGGER.try_with(|l| l.writer(category)).expect("Logger not set!")
-}
-
-pub fn get_logger() -> Rc<dyn Logger> { LOGGER.try_with(|l| l.clone()).expect("Logger not set!") }
-
-pub mod test {
-	use std::fmt::Arguments;
-	use std::rc::Rc;
-
-	use futures::future::LocalBoxFuture;
-
-	use crate::clone;
-	use crate::logging::{LogWriter, Logger};
-
 #[derive(Clone)]
-	pub struct TestLogger(Rc<dyn Fn(String) -> LocalBoxFuture<'static, ()>>);
+pub struct Logger(api::LogStrategy);
-	impl LogWriter for TestLogger {
+impl Logger {
-		fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()> {
+	pub fn new(strat: api::LogStrategy) -> Self { Self(strat) }
-			(self.0)(fmt.to_string())
+	pub fn log(&self, msg: impl AsRef<str>) { writeln!(self, "{}", msg.as_ref()) }
+	pub fn strat(&self) -> api::LogStrategy { self.0.clone() }
+	pub fn log_buf(&self, event: impl AsRef<str>, buf: &[u8]) {
+		if std::env::var("ORCHID_LOG_BUFFERS").is_ok_and(|v| !v.is_empty()) {
+			writeln!(self, "{}: [{}]", event.as_ref(), buf.iter().map(|b| format!("{b:02x}")).join(" "))
 		}
 	}
-	impl Logger for TestLogger {
+	pub fn write_fmt(&self, fmt: Arguments) {
-		fn strat(&self, _category: &str) -> orchid_api::LogStrategy { orchid_api::LogStrategy::Default }
+		match &self.0 {
-		fn writer(&self, _category: &str) -> std::rc::Rc<dyn LogWriter> { Rc::new(self.clone()) }
+			api::LogStrategy::Discard => (),
-	}
+			api::LogStrategy::StdErr => {
-	impl TestLogger {
+				stderr().write_fmt(fmt).expect("Could not write to stderr!");
-		pub fn new(f: impl AsyncFn(String) + 'static) -> Self {
+				stderr().flush().expect("Could not flush stderr")
-			let f = Rc::new(f);
+			},
-			Self(Rc::new(move |s| clone!(f; Box::pin(async move { f(s).await }))))
+			api::LogStrategy::File(f) => {
+				let mut file = (File::options().write(true).create(true).truncate(true).open(f))
+					.expect("Could not open logfile");
+				file.write_fmt(fmt).expect("Could not write to logfile");
+			},
 		}
 	}
-	impl Default for TestLogger {
-		fn default() -> Self { TestLogger::new(async |s| eprint!("{s}")) }
-	}
 }

orchid-base/src/msg.rs

@@ -6,8 +6,7 @@ use orchid_api_traits::{Decode, Encode};
 
 pub async fn send_msg(mut write: Pin<&mut impl AsyncWrite>, msg: &[u8]) -> io::Result<()> {
 	let mut len_buf = vec![];
-	let len_prefix = u32::try_from(msg.len()).expect("Message over 4GB not permitted on channel");
+	u32::try_from(msg.len()).unwrap().encode(Pin::new(&mut len_buf)).await;
-	len_prefix.encode_vec(&mut len_buf);
 	write.write_all(&len_buf).await?;
 	write.write_all(msg).await?;
 	write.flush().await
@@ -16,7 +15,7 @@ pub async fn send_msg(mut write: Pin<&mut impl AsyncWrite>, msg: &[u8]) -> io::R
 pub async fn recv_msg(mut read: Pin<&mut impl AsyncRead>) -> io::Result<Vec<u8>> {
 	let mut len_buf = [0u8; (u32::BITS / 8) as usize];
 	read.read_exact(&mut len_buf).await?;
-	let len = u32::decode(Pin::new(&mut &len_buf[..])).await?;
+	let len = u32::decode(Pin::new(&mut &len_buf[..])).await;
 	let mut msg = vec![0u8; len as usize];
 	read.read_exact(&mut msg).await?;
 	Ok(msg)

orchid-base/src/name.rs

@@ -12,60 +12,65 @@ use itertools::Itertools;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::interner::{IStr, IStrv, es, ev, is, iv};
+use crate::interner::{InternMarker, Interner, Tok};
 
 trait_set! {
 	/// Traits that all name iterators should implement
-	pub trait NameIter = Iterator<Item = IStr> + DoubleEndedIterator + ExactSizeIterator;
+	pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
 }
 
 /// A token path which may be empty. [VName] is the non-empty version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
-pub struct VPath(Vec<IStr>);
+pub struct VPath(Vec<Tok<String>>);
 impl VPath {
 	/// Collect segments into a vector
-	pub fn new(items: impl IntoIterator<Item = IStr>) -> Self { Self(items.into_iter().collect()) }
+	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Self {
+		Self(items.into_iter().collect())
+	}
 	/// Number of path segments
 	pub fn len(&self) -> usize { self.0.len() }
 	/// Whether there are any path segments. In other words, whether this is a
 	/// valid name
 	pub fn is_empty(&self) -> bool { self.len() == 0 }
 	/// Prepend some tokens to the path
-	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append some tokens to the path
-	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Partition the string by `::` namespace separators
-	pub async fn parse(s: &str) -> Self {
+	pub async fn parse(s: &str, i: &Interner) -> Self {
-		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(is)).await })
+		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(|s| i.i(s))).await })
 	}
 	/// Walk over the segments
-	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> { Box::new(self.0.iter().map(|s| &**s)) }
+	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
+		Box::new(self.0.iter().map(|s| s.as_str()))
+	}
 	/// Try to convert into non-empty version
 	pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
 	/// Add a token to the path. Since now we know that it can't be empty, turn it
 	/// into a name.
-	pub fn name_with_suffix(self, name: IStr) -> VName {
+	pub fn name_with_suffix(self, name: Tok<String>) -> VName {
 		VName(self.into_iter().chain([name]).collect())
 	}
 	/// Add a token to the beginning of the. Since now we know that it can't be
 	/// empty, turn it into a name.
-	pub fn name_with_prefix(self, name: IStr) -> VName {
+	pub fn name_with_prefix(self, name: Tok<String>) -> VName {
 		VName([name].into_iter().chain(self).collect())
 	}
 
 	/// Convert a fs path to a vpath
-	pub async fn from_path(path: &Path, ext: &str) -> Option<(Self, bool)> {
+	pub async fn from_path(path: &Path, ext: &str, i: &Interner) -> Option<(Self, bool)> {
-		async fn to_vpath(p: &Path) -> Option<VPath> {
+		async fn to_vpath(p: &Path, i: &Interner) -> Option<VPath> {
-			let tok_opt_v = join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(is)))).await;
+			let tok_opt_v =
+				join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(|s| i.i(s))))).await;
 			tok_opt_v.into_iter().collect::<Option<_>>().map(VPath)
 		}
 		match path.extension().map(|s| s.to_str()) {
-			Some(Some(s)) if s == ext => Some((to_vpath(&path.with_extension("")).await?, true)),
+			Some(Some(s)) if s == ext => Some((to_vpath(&path.with_extension(""), i).await?, true)),
-			None => Some((to_vpath(path).await?, false)),
+			None => Some((to_vpath(path, i).await?, false)),
 			Some(_) => None,
 		}
 	}
@@ -78,28 +83,30 @@ impl fmt::Display for VPath {
 		write!(f, "{}", self.str_iter().join("::"))
 	}
 }
-impl FromIterator<IStr> for VPath {
+impl FromIterator<Tok<String>> for VPath {
-	fn from_iter<T: IntoIterator<Item = IStr>>(iter: T) -> Self { Self(iter.into_iter().collect()) }
+	fn from_iter<T: IntoIterator<Item = Tok<String>>>(iter: T) -> Self {
+		Self(iter.into_iter().collect())
+	}
 }
 impl IntoIterator for VPath {
-	type Item = IStr;
+	type Item = Tok<String>;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
-impl Borrow<[IStr]> for VPath {
+impl Borrow<[Tok<String>]> for VPath {
-	fn borrow(&self) -> &[IStr] { &self.0[..] }
+	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
 }
 impl Deref for VPath {
-	type Target = [IStr];
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
 impl<T> Index<T> for VPath
-where [IStr]: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <[IStr] as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
-	fn index(&self, index: T) -> &Self::Output { &Borrow::<[IStr]>::borrow(self)[index] }
+	fn index(&self, index: T) -> &Self::Output { &Borrow::<[Tok<String>]>::borrow(self)[index] }
 }
 
 /// A mutable representation of a namespaced identifier of at least one segment.
@@ -109,43 +116,50 @@ where [IStr]: Index<T>
 /// See also [Sym] for the immutable representation, and [VPath] for possibly
 /// empty values
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct VName(Vec<IStr>);
+pub struct VName(Vec<Tok<String>>);
 impl VName {
 	/// Assert that the sequence isn't empty and wrap it in [VName] to represent
 	/// this invariant
-	pub fn new(items: impl IntoIterator<Item = IStr>) -> Result<Self, EmptyNameError> {
+	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
 		let data: Vec<_> = items.into_iter().collect();
 		if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
 	}
-	pub async fn deintern(name: impl IntoIterator<Item = api::TStr>) -> Result<Self, EmptyNameError> {
+	pub async fn deintern(
-		Self::new(join_all(name.into_iter().map(es)).await)
+		name: impl IntoIterator<Item = api::TStr>,
+		i: &Interner,
+	) -> Result<Self, EmptyNameError> {
+		Self::new(join_all(name.into_iter().map(|m| Tok::from_api(m, i))).await)
 	}
 	/// Unwrap the enclosed vector
-	pub fn into_vec(self) -> Vec<IStr> { self.0 }
+	pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
 	/// Get a reference to the enclosed vector
-	pub fn vec(&self) -> &Vec<IStr> { &self.0 }
+	pub fn vec(&self) -> &Vec<Tok<String>> { &self.0 }
 	/// Mutable access to the underlying vector. To ensure correct results, this
 	/// must never be empty.
-	pub fn vec_mut(&mut self) -> &mut Vec<IStr> { &mut self.0 }
+	pub fn vec_mut(&mut self) -> &mut Vec<Tok<String>> { &mut self.0 }
 	/// Intern the name and return a [Sym]
-	pub async fn to_sym(&self) -> Sym { Sym(iv(&self.0[..]).await) }
+	pub async fn to_sym(&self, i: &Interner) -> Sym { Sym(i.i(&self.0[..]).await) }
 	/// If this name has only one segment, return it
-	pub fn as_root(&self) -> Option<IStr> { self.0.iter().exactly_one().ok().cloned() }
+	pub fn as_root(&self) -> Option<Tok<String>> { self.0.iter().exactly_one().ok().cloned() }
 	/// Prepend the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Read a `::` separated namespaced name
-	pub async fn parse(s: &str) -> Result<Self, EmptyNameError> { Self::new(VPath::parse(s).await) }
+	pub async fn parse(s: &str, i: &Interner) -> Result<Self, EmptyNameError> {
-	pub async fn literal(s: &'static str) -> Self { Self::parse(s).await.expect("empty literal !?") }
+		Self::new(VPath::parse(s, i).await)
+	}
+	pub async fn literal(s: &'static str, i: &Interner) -> Self {
+		Self::parse(s, i).await.expect("empty literal !?")
+	}
 	/// Obtain an iterator over the segments of the name
-	pub fn iter(&self) -> impl Iterator<Item = IStr> + '_ { self.0.iter().cloned() }
+	pub fn iter(&self) -> impl Iterator<Item = Tok<String>> + '_ { self.0.iter().cloned() }
 }
 impl fmt::Debug for VName {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
@@ -156,22 +170,22 @@ impl fmt::Display for VName {
 	}
 }
 impl IntoIterator for VName {
-	type Item = IStr;
+	type Item = Tok<String>;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl<T> Index<T> for VName
-where [IStr]: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <[IStr] as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[IStr]> for VName {
+impl Borrow<[Tok<String>]> for VName {
-	fn borrow(&self) -> &[IStr] { self.0.borrow() }
+	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
 }
 impl Deref for VName {
-	type Target = [IStr];
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -179,9 +193,11 @@ impl Deref for VName {
 /// empty sequence
 #[derive(Debug, Copy, Clone, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct EmptyNameError;
-impl TryFrom<&[IStr]> for VName {
+impl TryFrom<&[Tok<String>]> for VName {
 	type Error = EmptyNameError;
-	fn try_from(value: &[IStr]) -> Result<Self, Self::Error> { Self::new(value.iter().cloned()) }
+	fn try_from(value: &[Tok<String>]) -> Result<Self, Self::Error> {
+		Self::new(value.iter().cloned())
+	}
 }
 
 /// An interned representation of a namespaced identifier.
@@ -190,35 +206,37 @@ impl TryFrom<&[IStr]> for VName {
 ///
 /// See also [VName]
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct Sym(IStrv);
+pub struct Sym(Tok<Vec<Tok<String>>>);
 impl Sym {
 	/// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
 	/// represent this invariant
-	pub async fn new(v: impl IntoIterator<Item = IStr>) -> Result<Self, EmptyNameError> {
+	pub async fn new(
+		v: impl IntoIterator<Item = Tok<String>>,
+		i: &Interner,
+	) -> Result<Self, EmptyNameError> {
 		let items = v.into_iter().collect_vec();
-		Self::from_tok(iv(&items).await)
+		Self::from_tok(i.i(&items).await)
 	}
-	/// Read a `::` separated namespaced name. Do not use this for statically
+	/// Read a `::` separated namespaced name.
-	/// known names, use the [sym] macro instead which is cached.
+	pub async fn parse(s: &str, i: &Interner) -> Result<Self, EmptyNameError> {
-	pub async fn parse(s: &str) -> Result<Self, EmptyNameError> {
+		Ok(Sym(i.i(&VName::parse(s, i).await?.into_vec()).await))
-		Ok(Sym(iv(&VName::parse(s).await?.into_vec()).await))
 	}
 	/// Assert that a token isn't empty, and wrap it in a [Sym]
-	pub fn from_tok(t: IStrv) -> Result<Self, EmptyNameError> {
+	pub fn from_tok(t: Tok<Vec<Tok<String>>>) -> Result<Self, EmptyNameError> {
 		if t.is_empty() { Err(EmptyNameError) } else { Ok(Self(t)) }
 	}
 	/// Grab the interner token
-	pub fn tok(&self) -> IStrv { self.0.clone() }
+	pub fn tok(&self) -> Tok<Vec<Tok<String>>> { self.0.clone() }
 	/// Get a number unique to this name suitable for arbitrary ordering.
-	pub fn id(&self) -> NonZeroU64 { self.0.to_api().0 }
+	pub fn id(&self) -> NonZeroU64 { self.0.to_api().get_id() }
 	/// Extern the sym for editing
 	pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
-	pub async fn from_api(marker: api::TStrv) -> Sym {
+	pub async fn from_api(marker: api::TStrv, i: &Interner) -> Sym {
-		Self::from_tok(ev(marker).await).expect("Empty sequence found for serialized Sym")
+		Self::from_tok(Tok::from_api(marker, i).await).expect("Empty sequence found for serialized Sym")
 	}
 	pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
-	pub async fn suffix(&self, tokv: impl IntoIterator<Item = IStr>) -> Sym {
+	pub async fn suffix(&self, tokv: impl IntoIterator<Item = Tok<String>>, i: &Interner) -> Sym {
-		Self::new(self.0.iter().cloned().chain(tokv)).await.unwrap()
+		Self::new(self.0.iter().cloned().chain(tokv), i).await.unwrap()
 	}
 }
 impl fmt::Debug for Sym {
@@ -230,17 +248,17 @@ impl fmt::Display for Sym {
 	}
 }
 impl<T> Index<T> for Sym
-where [IStr]: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <[IStr] as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[IStr]> for Sym {
+impl Borrow<[Tok<String>]> for Sym {
-	fn borrow(&self) -> &[IStr] { &self.0[..] }
+	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
 }
 impl Deref for Sym {
-	type Target = [IStr];
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -248,14 +266,16 @@ impl Deref for Sym {
 /// handled together in datastructures. The names can never be empty
 #[allow(clippy::len_without_is_empty)] // never empty
 pub trait NameLike:
-	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[IStr]>
+	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[Tok<String>]>
 {
 	/// Convert into held slice
-	fn as_slice(&self) -> &[IStr] { Borrow::<[IStr]>::borrow(self) }
+	fn as_slice(&self) -> &[Tok<String>] { Borrow::<[Tok<String>]>::borrow(self) }
 	/// Get iterator over tokens
 	fn segs(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
 	/// Get iterator over string segments
-	fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ { self.as_slice().iter().map(|t| &**t) }
+	fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ {
+		self.as_slice().iter().map(|t| t.as_str())
+	}
 	/// Fully resolve the name for printing
 	#[must_use]
 	fn to_strv(&self) -> Vec<String> { self.segs().map(|s| s.to_string()).collect() }
@@ -266,19 +286,19 @@ pub trait NameLike:
 		NonZeroUsize::try_from(self.segs().count()).expect("NameLike never empty")
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
-	fn split_first_seg(&self) -> (IStr, &[IStr]) {
+	fn split_first_seg(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some
-	fn split_last_seg(&self) -> (IStr, &[IStr]) {
+	fn split_last_seg(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Get the first element
-	fn first_seg(&self) -> IStr { self.split_first_seg().0 }
+	fn first_seg(&self) -> Tok<String> { self.split_first_seg().0 }
 	/// Get the last element
-	fn last_seg(&self) -> IStr { self.split_last_seg().0 }
+	fn last_seg(&self) -> Tok<String> { self.split_last_seg().0 }
 }
 
 impl NameLike for Sym {}
@@ -286,27 +306,19 @@ impl NameLike for VName {}
 
 /// Create a [Sym] literal.
 ///
-/// The name and its components will be cached in a thread-local static so
+/// Both the name and its components will be cached in a thread-local static so
 /// that subsequent executions of the expression only incur an Arc-clone for
 /// cloning the token.
 #[macro_export]
 macro_rules! sym {
-  ($seg1:tt $( :: $seg:tt)*) => {
+  ($seg1:tt $( :: $seg:tt)* ; $i:expr) => {
-		$crate::tl_cache!(async $crate::name::Sym : {
 		$crate::name::Sym::from_tok(
-				$crate::interner::iv(&[
+			$i.i(&[
-					$crate::interner::is($crate::sym!(@SEG $seg1)).await
+				$i.i(stringify!($seg1)).await
-					$( , $crate::interner::is($crate::sym!(@SEG $seg)).await )*
+				$( , $i.i(stringify!($seg)).await )*
 			])
 			.await
 		).unwrap()
-		})
-  };
-	(@SEG [ $($data:tt)* ]) => {
-		stringify!($($data)*)
-	};
-	(@SEG $data:tt) => {
-		stringify!($data)
   };
 }
 
@@ -315,13 +327,11 @@ macro_rules! sym {
 /// The components are interned much like in [sym].
 #[macro_export]
 macro_rules! vname {
-  ($seg1:tt $( :: $seg:tt)*) => {
+  ($seg1:tt $( :: $seg:tt)* ; $i:expr) => {
-		$crate::tl_cache!(async $crate::name::VName : {
     $crate::name::VName::new([
-				$crate::interner::is(stringify!($seg1)).await
+      $i.i(stringify!($seg1)).await
-				$( , $crate::interner::is(stringify!($seg)).await )*
+      $( , $i.i(stringify!($seg)).await )*
     ]).unwrap()
-		})
 	};
 }
 
@@ -330,13 +340,11 @@ macro_rules! vname {
 /// The components are interned much like in [sym].
 #[macro_export]
 macro_rules! vpath {
-  ($seg1:tt $( :: $seg:tt)*) => {
+  ($seg1:tt $( :: $seg:tt)+ ; $i:expr) => {
-		$crate::tl_cache!(async $crate::name::VPath : {
     $crate::name::VPath(vec![
-				$crate::interner::is(stringify!($seg1)).await
+      $i.i(stringify!($seg1)).await
-				$( , $crate::interner::is(stringify!($seg)).await )*
+      $( , $i.i(stringify!($seg)).await )+
 		])
-		})
 	};
   () => {
     $crate::name::VPath(vec![])
@@ -344,46 +352,42 @@ macro_rules! vpath {
 }
 
 #[cfg(test)]
-pub mod test {
+mod test {
 	use std::borrow::Borrow;
 
-	use orchid_api_traits::spin_on;
+	use test_executors::spin_on;
 
 	use super::{NameLike, Sym, VName};
-	use crate::interner::local_interner::local_interner;
+	use crate::interner::{Interner, Tok};
-	use crate::interner::{IStr, is, with_interner};
 	use crate::name::VPath;
 
 	#[test]
-	pub fn recur() {
+	fn recur() {
-		spin_on(with_interner(local_interner(), async {
+		spin_on(async {
-			let myname = vname!(foo::bar);
+			let i = Interner::new_master();
-			let _borrowed_slice: &[IStr] = myname.borrow();
+			let myname = vname!(foo::bar; i);
-			let _deref_pathslice: &[IStr] = &myname;
+			let _borrowed_slice: &[Tok<String>] = myname.borrow();
-			let _as_slice_out: &[IStr] = myname.as_slice();
+			let _deref_pathslice: &[Tok<String>] = &myname;
-		}))
+			let _as_slice_out: &[Tok<String>] = myname.as_slice();
+		})
 	}
 
-	/// Tests that literals are correctly interned as equal
 	#[test]
-	pub fn literals() {
+	fn literals() {
-		spin_on(with_interner(local_interner(), async {
+		spin_on(async {
+			let i = Interner::new_master();
 			assert_eq!(
-				sym!(foo::bar::baz),
+				sym!(foo::bar::baz; i),
-				Sym::new([is("foo").await, is("bar").await, is("baz").await]).await.unwrap()
+				Sym::new([i.i("foo").await, i.i("bar").await, i.i("baz").await], &i).await.unwrap()
 			);
 			assert_eq!(
-				sym!(foo::bar::[|>]),
+				vname!(foo::bar::baz; i),
-				Sym::new([is("foo").await, is("bar").await, is("|>").await]).await.unwrap()
+				VName::new([i.i("foo").await, i.i("bar").await, i.i("baz").await]).unwrap()
 			);
 			assert_eq!(
-				vname!(foo::bar::baz),
+				vpath!(foo::bar::baz; i),
-				VName::new([is("foo").await, is("bar").await, is("baz").await]).unwrap()
+				VPath::new([i.i("foo").await, i.i("bar").await, i.i("baz").await])
 			);
-			assert_eq!(
+		})
-				{ vpath!(foo::bar::baz) },
-				VPath::new([is("foo").await, is("bar").await, is("baz").await])
-			);
-		}))
 	}
 }

orchid-base/src/number.rs

@@ -4,7 +4,7 @@ use std::ops::Range;
 use ordered_float::NotNan;
 
 use crate::error::{OrcErrv, mk_errv};
-use crate::interner::is;
+use crate::interner::Interner;
 use crate::location::SrcRange;
 use crate::name::Sym;
 
@@ -55,9 +55,14 @@ pub struct NumError {
 	pub kind: NumErrorKind,
 }
 
-pub async fn num_to_errv(NumError { kind, range }: NumError, offset: u32, source: &Sym) -> OrcErrv {
+pub async fn num_to_errv(
+	NumError { kind, range }: NumError,
+	offset: u32,
+	source: &Sym,
+	i: &Interner,
+) -> OrcErrv {
 	mk_errv(
-		is("Failed to parse number").await,
+		i.i("Failed to parse number").await,
 		match kind {
 			NumErrorKind::NaN => "NaN emerged during parsing",
 			NumErrorKind::InvalidDigit => "non-digit character encountered",

orchid-base/src/parse.rs

@@ -7,16 +7,31 @@ use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::api;
-use crate::error::{OrcErrv, OrcRes, mk_errv, report};
+use crate::error::{OrcErrv, OrcRes, Reporter, mk_errv};
 use crate::format::{FmtCtx, FmtUnit, Format, fmt};
-use crate::interner::{IStr, es, is};
+use crate::interner::{Interner, Tok};
 use crate::location::SrcRange;
 use crate::name::{Sym, VName, VPath};
 use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_fmt, ttv_range};
 
+pub trait ParseCtx {
+	#[must_use]
+	fn i(&self) -> &Interner;
+	#[must_use]
+	fn rep(&self) -> &Reporter;
+}
+pub struct ParseCtxImpl<'a> {
+	pub i: &'a Interner,
+	pub r: &'a Reporter,
+}
+impl ParseCtx for ParseCtxImpl<'_> {
+	fn i(&self) -> &Interner { self.i }
+	fn rep(&self) -> &Reporter { self.r }
+}
+
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
 pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
-pub fn op_char(c: char) -> bool { !name_char(c) && !unrep_space(c) && !"()[]{}\\".contains(c) }
+pub fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}\\".contains(c) }
 pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 
 /// A cheaply copiable subsection of a document that holds onto context data and
@@ -88,22 +103,22 @@ impl<A: ExprRepr, X: ExtraTok> Format for Snippet<'_, A, X> {
 
 #[derive(Clone, Debug)]
 pub struct Comment {
-	pub text: IStr,
+	pub text: Tok<String>,
 	pub sr: SrcRange,
 }
 impl Comment {
 	// XXX: which of these four are actually used?
-	pub async fn from_api(c: &api::Comment, src: Sym) -> Self {
+	pub async fn from_api(c: &api::Comment, src: Sym, i: &Interner) -> Self {
-		Self { text: es(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
+		Self { text: i.ex(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
 	}
-	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>) -> Option<Self> {
+	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, i: &Interner) -> Option<Self> {
 		match &tk.tok {
-			Token::Comment(text) => Some(Self { text: text.clone(), sr: tk.sr.clone() }),
+			Token::Comment(text) => Some(Self { text: i.i(&**text).await, sr: tk.sr.clone() }),
 			_ => None,
 		}
 	}
 	pub fn to_tk<R: ExprRepr, X: ExtraTok>(&self) -> TokTree<R, X> {
-		TokTree { tok: Token::Comment(self.text.clone()), sr: self.sr.clone() }
+		TokTree { tok: Token::Comment(self.text.rc().clone()), sr: self.sr.clone() }
 	}
 	pub fn to_api(&self) -> api::Comment {
 		api::Comment { range: self.sr.range(), text: self.text.to_api() }
@@ -115,6 +130,7 @@ impl fmt::Display for Comment {
 }
 
 pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
+	ctx: &impl ParseCtx,
 	snip: Snippet<'a, A, X>,
 ) -> Vec<Parsed<'a, Vec<Comment>, A, X>> {
 	let mut items = Vec::new();
@@ -129,10 +145,9 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 			None => comments.extend(line.cur),
 			Some(i) => {
 				let (cmts, tail) = line.split_at(i);
-				let comments = join_all(
+				let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
-					(comments.drain(..).chain(cmts.cur))
+					Comment::from_tk(t, ctx.i()).await.expect("All are comments checked above")
-						.map(|t| async { Comment::from_tk(t).await.expect("All are comments checked above") }),
+				}))
-				)
 				.await;
 				items.push(Parsed { output: comments, tail });
 			},
@@ -142,21 +157,26 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 }
 
 pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
+	ctx: &impl ParseCtx,
 	snip: Snippet<'a, A, X>,
 ) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
 	match snip.skip_fluff().pop_front() {
 		Some((output, tail)) => Ok(Parsed { output, tail }),
-		None =>
+		None => Err(mk_errv(
-			Err(mk_errv(is("Unexpected end").await, "Line ends abruptly; more tokens were expected", [
+			ctx.i().i("Unexpected end").await,
-				snip.sr(),
+			"Line ends abruptly; more tokens were expected",
-			])),
+			[snip.sr()],
+		)),
 	}
 }
 
-pub async fn expect_end(snip: Snippet<'_, impl ExprRepr, impl ExtraTok>) -> OrcRes<()> {
+pub async fn expect_end(
+	ctx: &impl ParseCtx,
+	snip: Snippet<'_, impl ExprRepr, impl ExtraTok>,
+) -> OrcRes<()> {
 	match snip.skip_fluff().get(0) {
 		Some(surplus) => Err(mk_errv(
-			is("Extra code after end of line").await,
+			ctx.i().i("Extra code after end of line").await,
 			"Code found after the end of the line",
 			[surplus.sr.pos()],
 		)),
@@ -165,26 +185,28 @@ pub async fn expect_end(snip: Snippet<'_, impl ExprRepr, impl ExtraTok>) -> OrcR
 }
 
 pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
+	ctx: &impl ParseCtx,
 	snip: Snippet<'a, A, X>,
-	tok: IStr,
+	tok: Tok<String>,
 ) -> ParseRes<'a, (), A, X> {
-	let Parsed { output: head, tail } = try_pop_no_fluff(snip).await?;
+	let Parsed { output: head, tail } = try_pop_no_fluff(ctx, snip).await?;
 	match &head.tok {
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
 		t => Err(mk_errv(
-			is("Expected specific keyword").await,
+			ctx.i().i("Expected specific keyword").await,
-			format!("Expected {tok} but found {:?}", fmt(t).await),
+			format!("Expected {tok} but found {:?}", fmt(t, ctx.i()).await),
 			[head.sr()],
 		)),
 	}
 }
 
 pub async fn token_errv<A: ExprRepr, X: ExtraTok>(
+	ctx: &impl ParseCtx,
 	tok: &TokTree<A, X>,
 	description: &'static str,
 	message: impl FnOnce(&str) -> String,
 ) -> OrcErrv {
-	mk_errv(is(description).await, message(&fmt(tok).await), [tok.sr.pos()])
+	mk_errv(ctx.i().i(description).await, message(&fmt(tok, ctx.i()).await), [tok.sr.pos()])
 }
 
 pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
@@ -195,27 +217,33 @@ pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
 pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
 
 pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
+	ctx: &impl ParseCtx,
 	tail: Snippet<'a, A, X>,
 ) -> ParseRes<'a, Vec<Import>, A, X> {
 	let Some((tt, tail)) = tail.skip_fluff().pop_front() else {
 		return Err(mk_errv(
-			is("Expected token").await,
+			ctx.i().i("Expected token").await,
 			"Expected a name, a parenthesized list of names, or a globstar.",
 			[tail.sr().pos()],
 		));
 	};
-	let ret = rec(tt).await;
+	let ret = rec(tt, ctx).await;
 	#[allow(clippy::type_complexity)] // it's an internal function
 	pub async fn rec<A: ExprRepr, X: ExtraTok>(
 		tt: &TokTree<A, X>,
-	) -> OrcRes<Vec<(Vec<IStr>, Option<IStr>, SrcRange)>> {
+		ctx: &impl ParseCtx,
+	) -> OrcRes<Vec<(Vec<Tok<String>>, Option<Tok<String>>, SrcRange)>> {
 		let ttpos = tt.sr.pos();
 		match &tt.tok {
 			Token::NS(ns, body) => {
 				if !ns.starts_with(name_start) {
-					report(mk_errv(is("Unexpected name prefix").await, "Only names can precede ::", [ttpos]))
+					ctx.rep().report(mk_errv(
+						ctx.i().i("Unexpected name prefix").await,
+						"Only names can precede ::",
+						[ttpos],
+					))
 				};
-				let out = Box::pin(rec(body)).await?;
+				let out = Box::pin(rec(body, ctx)).await?;
 				Ok(out.into_iter().update(|i| i.0.push(ns.clone())).collect_vec())
 			},
 			Token::Name(ntok) => {
@@ -227,19 +255,21 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 				let mut o = Vec::new();
 				let mut body = Snippet::new(tt, b);
 				while let Some((output, tail)) = body.pop_front() {
-					match rec(output).boxed_local().await {
+					match rec(output, ctx).boxed_local().await {
 						Ok(names) => o.extend(names),
-						Err(e) => report(e),
+						Err(e) => ctx.rep().report(e),
 					}
 					body = tail;
 				}
 				Ok(o)
 			},
-			t => Err(mk_errv(
+			t => {
-				is("Unrecognized name end").await,
+				return Err(mk_errv(
-				format!("Names cannot end with {:?} tokens", fmt(t).await),
+					ctx.i().i("Unrecognized name end").await,
+					format!("Names cannot end with {:?} tokens", fmt(t, ctx.i()).await),
 					[ttpos],
-			)),
+				));
+			},
 		}
 	}
 	ret.map(|output| {
@@ -255,7 +285,7 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 #[derive(Debug, Clone)]
 pub struct Import {
 	pub path: VPath,
-	pub name: Option<IStr>,
+	pub name: Option<Tok<String>>,
 	pub sr: SrcRange,
 }
 impl Import {
@@ -266,14 +296,14 @@ impl Import {
 			None => self.path.into_name().expect("Import cannot be empty"),
 		}
 	}
-	pub fn new(sr: SrcRange, path: VPath, name: IStr) -> Self {
+	pub fn new(sr: SrcRange, path: VPath, name: Tok<String>) -> Self {
 		Import { path, name: Some(name), sr }
 	}
 	pub fn new_glob(sr: SrcRange, path: VPath) -> Self { Import { path, name: None, sr } }
 }
 impl Display for Import {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| &**t))
+		write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str()))
 	}
 }
 

orchid-base/src/reqnot.rs

@@ -1,583 +1,342 @@
 use std::cell::RefCell;
+use std::future::Future;
 use std::marker::PhantomData;
-use std::pin::{Pin, pin};
+use std::mem;
-use std::rc::Rc;
+use std::ops::{BitAnd, Deref};
-use std::{io, mem};
+use std::pin::Pin;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
 
-use async_fn_stream::try_stream;
-use bound::Bound;
 use derive_destructure::destructure;
-use futures::channel::mpsc::{self, Receiver, Sender, channel};
+use dyn_clone::{DynClone, clone_box};
-use futures::channel::oneshot;
+use futures::channel::mpsc;
 use futures::future::LocalBoxFuture;
-use futures::lock::{Mutex, MutexGuard};
+use futures::lock::Mutex;
-use futures::{
+use futures::{SinkExt, StreamExt};
-	AsyncRead, AsyncWrite, AsyncWriteExt, FutureExt, SinkExt, Stream, StreamExt, stream_select,
-};
 use hashbrown::HashMap;
-use orchid_api_traits::{Decode, Encode, Request, UnderRoot};
+use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
+use trait_set::trait_set;
 
-use crate::future_debug::{PanicOnDrop, assert_no_drop};
+use crate::clone;
-use crate::localset::LocalSet;
+use crate::logging::Logger;
 
-#[must_use = "Receipts indicate that a required action has been performed within a function. \
-	Most likely this should be returned somewhere."]
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
-impl Receipt<'_> {
+
-	/// Only call this function from a custom implementation of [RepWriter]
+trait_set! {
-	pub fn _new() -> Self { Self(PhantomData) }
+	pub trait SendFn<T: MsgSet> =
+		for<'a> FnMut(&'a [u8], ReqNot<T>) -> LocalBoxFuture<'a, ()>
+		+ DynClone + 'static;
+	pub trait ReqFn<T: MsgSet> =
+		for<'a> FnMut(RequestHandle<'a, T>, <T::In as Channel>::Req)
+			-> LocalBoxFuture<'a, Receipt<'a>>
+		+ DynClone + 'static;
+	pub trait NotifFn<T: MsgSet> =
+		FnMut(<T::In as Channel>::Notif, ReqNot<T>) -> LocalBoxFuture<'static, ()>
+		+ DynClone + 'static;
 }
 
-/// Write guard to outbound for the purpose of serializing a request. Only one
+fn get_id(message: &[u8]) -> (u64, &[u8]) {
-/// can exist at a time. Dropping this object should panic.
+	(u64::from_be_bytes(message[..8].to_vec().try_into().unwrap()), &message[8..])
-pub trait ReqWriter<'a> {
-	/// Access to the underlying channel. This may be buffered.
-	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
-	/// Finalize the request, release the outbound channel, then queue for the
-	/// reply on the inbound channel.
-	fn send(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepReader<'a> + 'a>>>;
 }
 
-/// Write guard to inbound for the purpose of deserializing a reply. While held,
+pub trait ReqHandlish {
-/// no inbound requests or other replies can be processed.
+	fn defer(&self, cb: impl Future<Output = ()> + 'static)
-///
+	where Self: Sized {
-/// Dropping this object should panic even if [RepReader::finish] returns
+		self.defer_objsafe(Box::pin(cb));
-/// synchronously, because the API isn't cancellation safe in general so it is a
-/// programmer error in all cases to drop an object related to it without proper
-/// cleanup.
-pub trait RepReader<'a> {
-	/// Access to the underlying channel. The length of the message is inferred
-	/// from the number of bytes read so this must not be buffered.
-	fn reader(&mut self) -> Pin<&mut dyn AsyncRead>;
-	/// Finish reading the request
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, ()>;
 	}
-
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>);
-/// Write guard to outbound for the purpose of serializing a notification.
-///
-/// Dropping this object should panic for the same reason [RepReader] panics
-pub trait MsgWriter<'a> {
-	/// Access to the underlying channel. This may be buffered.
-	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
-	/// Send the notification
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<()>>;
-}
-
-/// For initiating outbound requests and notifications
-pub trait Client {
-	fn start_request(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn ReqWriter<'_> + '_>>>;
-	fn start_notif(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn MsgWriter<'_> + '_>>>;
-}
-
-impl<T: Client + ?Sized> ClientExt for T {}
-/// Extension trait with convenience methods that handle outbound request and
-/// notif lifecycle and typing
-#[allow(async_fn_in_trait)]
-pub trait ClientExt: Client {
-	async fn request<T: Request + UnderRoot<Root: Encode>>(&self, t: T) -> io::Result<T::Response> {
-		let mut req = self.start_request().await?;
-		t.into_root().encode(req.writer().as_mut()).await?;
-		let mut rep = req.send().await?;
-		let response = T::Response::decode(rep.reader()).await;
-		rep.finish().await;
-		response
-	}
-	async fn notify<T: UnderRoot<Root: Encode>>(&self, t: T) -> io::Result<()> {
-		let mut notif = self.start_notif().await?;
-		t.into_root().encode(notif.writer().as_mut()).await?;
-		notif.finish().await?;
-		Ok(())
-	}
-}
-
-pub trait ReqReader<'a> {
-	fn reader(&mut self) -> Pin<&mut dyn AsyncRead>;
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>>;
-}
-impl<'a, T: ReqReader<'a> + ?Sized> ReqReaderExt<'a> for T {}
-#[allow(async_fn_in_trait)]
-pub trait ReqReaderExt<'a>: ReqReader<'a> {
-	async fn read_req<R: Decode>(&mut self) -> io::Result<R> { R::decode(self.reader()).await }
-	async fn reply<R: Request>(
-		self: Box<Self>,
-		req: impl Evidence<R>,
-		rep: &R::Response,
-	) -> io::Result<Receipt<'a>> {
-		self.finish().await.reply(req, rep).await
-	}
-	async fn start_reply(self: Box<Self>) -> io::Result<Box<dyn RepWriter<'a> + 'a>> {
-		self.finish().await.start_reply().await
-	}
-}
-
-pub trait ReqHandle<'a> {
-	fn start_reply(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>>;
-}
-impl<'a, T: ReqHandle<'a> + ?Sized> ReqHandleExt<'a> for T {}
-#[allow(async_fn_in_trait)]
-pub trait ReqHandleExt<'a>: ReqHandle<'a> {
-	async fn reply<Req: Request>(
-		self: Box<Self>,
-		_: impl Evidence<Req>,
-		rep: &Req::Response,
-	) -> io::Result<Receipt<'a>> {
-		let mut reply = self.start_reply().await?;
-		rep.encode(reply.writer()).await?;
-		reply.finish().await
-	}
-}
-
-pub trait RepWriter<'a> {
-	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Receipt<'a>>>;
-}
-
-pub trait MsgReader<'a> {
-	fn reader(&mut self) -> Pin<&mut dyn AsyncRead>;
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, ()>;
-}
-impl<'a, T: ?Sized + MsgReader<'a>> MsgReaderExt<'a> for T {}
-#[allow(async_fn_in_trait)]
-pub trait MsgReaderExt<'a>: MsgReader<'a> {
-	async fn read<N: Decode>(mut self: Box<Self>) -> io::Result<N> {
-		let n = N::decode(self.reader()).await;
-		self.finish().await;
-		n
-	}
-}
-
-/// A form of [Evidence] that doesn't require the value to be kept around
-pub struct Witness<T>(PhantomData<T>);
-impl<T> Witness<T> {
-	pub fn of(_: &T) -> Self { Self(PhantomData) }
-}
-impl<T> Copy for Witness<T> {}
-impl<T> Clone for Witness<T> {
-	fn clone(&self) -> Self { *self }
-}
-
-/// A proxy for the type of a value either previously saved into a [Witness] or
-/// still available.
-pub trait Evidence<T> {}
-impl<T> Evidence<T> for &'_ T {}
-impl<T> Evidence<T> for Witness<T> {}
-
-type IoRef<T> = Pin<Box<T>>;
-type IoLock<T> = Rc<Mutex<Pin<Box<T>>>>;
-type IoGuard<T> = Bound<MutexGuard<'static, Pin<Box<T>>>, IoLock<T>>;
-
-/// An incoming request. This holds a lock on the ingress channel.
-pub struct IoReqReader<'a> {
-	prefix: &'a [u8],
-	read: IoGuard<dyn AsyncRead>,
-	write: &'a Mutex<IoRef<dyn AsyncWrite>>,
-}
-impl<'a> ReqReader<'a> for IoReqReader<'a> {
-	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { self.read.as_mut() }
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>> {
-		Box::pin(async {
-			Box::new(IoReqHandle { prefix: self.prefix, write: self.write }) as Box<dyn ReqHandle<'a>>
-		})
-	}
-}
-pub struct IoReqHandle<'a> {
-	prefix: &'a [u8],
-	write: &'a Mutex<IoRef<dyn AsyncWrite>>,
-}
-impl<'a> ReqHandle<'a> for IoReqHandle<'a> {
-	fn start_reply(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>> {
-		Box::pin(async move {
-			let mut write = self.write.lock().await;
-			write.as_mut().write_all(self.prefix).await?;
-			Ok(Box::new(IoRepWriter { write }) as Box<dyn RepWriter<'a>>)
-		})
-	}
-}
-pub struct IoRepWriter<'a> {
-	write: MutexGuard<'a, IoRef<dyn AsyncWrite>>,
-}
-impl<'a> RepWriter<'a> for IoRepWriter<'a> {
-	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.write.as_mut() }
-	fn finish(mut self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Receipt<'a>>> {
-		Box::pin(async move {
-			self.writer().flush().await?;
-			Ok(Receipt(PhantomData))
-		})
-	}
-}
-
-pub struct IoMsgReader<'a> {
-	_pd: PhantomData<&'a mut ()>,
-	read: IoGuard<dyn AsyncRead>,
-}
-impl<'a> MsgReader<'a> for IoMsgReader<'a> {
-	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { self.read.as_mut() }
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'static, ()> { Box::pin(async {}) }
-}
-
-#[derive(Debug)]
-struct ReplySub {
-	id: u64,
-	ack: oneshot::Sender<()>,
-	cb: oneshot::Sender<IoGuard<dyn AsyncRead>>,
-}
-
-struct IoClient {
-	output: IoLock<dyn AsyncWrite>,
-	id: Rc<RefCell<u64>>,
-	subscribe: Rc<Sender<ReplySub>>,
-}
-impl IoClient {
-	fn new(output: IoLock<dyn AsyncWrite>) -> (Receiver<ReplySub>, Self) {
-		let (req, rep) = mpsc::channel(0);
-		(rep, Self { output, id: Rc::new(RefCell::new(0)), subscribe: Rc::new(req) })
-	}
-	async fn lock_out(&self) -> IoGuard<dyn AsyncWrite> {
-		Bound::async_new(self.output.clone(), async |o| o.lock().await).await
-	}
-}
-impl Client for IoClient {
-	fn start_notif(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn MsgWriter<'_> + '_>>> {
-		Box::pin(async {
-			let drop_g = assert_no_drop("Notif future dropped");
-			let mut o = self.lock_out().await;
-			0u64.encode(o.as_mut()).await?;
-			drop_g.defuse();
-			Ok(Box::new(IoNotifWriter { o, drop_g: assert_no_drop("Notif writer dropped") })
-				as Box<dyn MsgWriter>)
-		})
-	}
-	fn start_request(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn ReqWriter<'_> + '_>>> {
-		Box::pin(async {
-			let id = {
-				let mut id_g = self.id.borrow_mut();
-				*id_g += 1;
-				*id_g
-			};
-			let (cb, reply) = oneshot::channel();
-			let (ack, got_ack) = oneshot::channel();
-			let drop_g = assert_no_drop("Request future dropped");
-			self.subscribe.as_ref().clone().send(ReplySub { id, ack, cb }).await.unwrap();
-			got_ack.await.unwrap();
-			let mut w = self.lock_out().await;
-			id.encode(w.as_mut()).await?;
-			drop_g.defuse();
-			Ok(Box::new(IoReqWriter {
-				reply,
-				w,
-				drop_g: assert_no_drop("Request reader dropped without reply"),
-			}) as Box<dyn ReqWriter>)
-		})
-	}
-}
-
-struct IoReqWriter {
-	reply: oneshot::Receiver<IoGuard<dyn AsyncRead>>,
-	w: IoGuard<dyn AsyncWrite>,
-	drop_g: PanicOnDrop,
-}
-impl<'a> ReqWriter<'a> for IoReqWriter {
-	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.w.as_mut() }
-	fn send(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepReader<'a> + 'a>>> {
-		Box::pin(async {
-			let Self { reply, mut w, drop_g } = *self;
-			w.flush().await?;
-			mem::drop(w);
-			let i = reply.await.expect("Client dropped before reply received");
-			drop_g.defuse();
-			Ok(Box::new(IoRepReader {
-				i,
-				drop_g: assert_no_drop("Reply reader dropped without finishing"),
-			}) as Box<dyn RepReader>)
-		})
-	}
-}
-
-struct IoRepReader {
-	i: IoGuard<dyn AsyncRead>,
-	drop_g: PanicOnDrop,
-}
-impl<'a> RepReader<'a> for IoRepReader {
-	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { self.i.as_mut() }
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'static, ()> {
-		Box::pin(async { self.drop_g.defuse() })
 }
+impl ReqHandlish for &'_ dyn ReqHandlish {
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) { (**self).defer_objsafe(val) }
 }
 
 #[derive(destructure)]
-struct IoNotifWriter {
+pub struct RequestHandle<'a, MS: MsgSet> {
-	o: IoGuard<dyn AsyncWrite>,
+	defer: RefCell<Vec<Pin<Box<dyn Future<Output = ()>>>>>,
-	drop_g: PanicOnDrop,
+	fulfilled: AtomicBool,
+	id: u64,
+	_reqlt: PhantomData<&'a mut ()>,
+	parent: ReqNot<MS>,
 }
-impl<'a> MsgWriter<'a> for IoNotifWriter {
+impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
-	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.o.as_mut() }
+	fn new(parent: ReqNot<MS>, id: u64) -> Self {
-	fn finish(mut self: Box<Self>) -> LocalBoxFuture<'static, io::Result<()>> {
+		Self { defer: RefCell::default(), fulfilled: false.into(), _reqlt: PhantomData, parent, id }
+	}
+	pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
+	pub async fn handle<U: Request>(&self, _: &U, rep: &U::Response) -> Receipt<'a> {
+		self.respond(rep).await
+	}
+	pub fn will_handle_as<U: Request>(&self, _: &U) -> ReqTypToken<U> { ReqTypToken(PhantomData) }
+	pub async fn handle_as<U: Request>(&self, _: ReqTypToken<U>, rep: &U::Response) -> Receipt<'a> {
+		self.respond(rep).await
+	}
+	pub async fn respond(&self, response: &impl Encode) -> Receipt<'a> {
+		assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id);
+		let mut buf = (!self.id).to_be_bytes().to_vec();
+		response.encode(Pin::new(&mut buf)).await;
+		let mut send = clone_box(&*self.reqnot().0.lock().await.send);
+		(send)(&buf, self.parent.clone()).await;
+		let deferred = mem::take(&mut *self.defer.borrow_mut());
+		for item in deferred {
+			item.await
+		}
+		Receipt(PhantomData)
+	}
+}
+impl<MS: MsgSet> ReqHandlish for RequestHandle<'_, MS> {
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) {
+		self.defer.borrow_mut().push(val)
+	}
+}
+impl<MS: MsgSet> Drop for RequestHandle<'_, MS> {
+	fn drop(&mut self) {
+		let done = self.fulfilled.load(Ordering::Relaxed);
+		debug_assert!(done, "Request {} dropped without response", self.id)
+	}
+}
+
+pub struct ReqTypToken<T>(PhantomData<T>);
+
+pub struct ReqNotData<T: MsgSet> {
+	id: u64,
+	send: Box<dyn SendFn<T>>,
+	notif: Box<dyn NotifFn<T>>,
+	req: Box<dyn ReqFn<T>>,
+	responses: HashMap<u64, mpsc::Sender<Vec<u8>>>,
+}
+
+/// Wraps a raw message buffer to save on copying.
+/// Dereferences to the tail of the message buffer, cutting off the ID
+#[derive(Debug, Clone)]
+pub struct RawReply(Vec<u8>);
+impl Deref for RawReply {
+	type Target = [u8];
+	fn deref(&self) -> &Self::Target { get_id(&self.0[..]).1 }
+}
+
+pub struct ReqNot<T: MsgSet>(Arc<Mutex<ReqNotData<T>>>, Logger);
+impl<T: MsgSet> ReqNot<T> {
+	pub fn new(
+		logger: Logger,
+		send: impl SendFn<T>,
+		notif: impl NotifFn<T>,
+		req: impl ReqFn<T>,
+	) -> Self {
+		Self(
+			Arc::new(Mutex::new(ReqNotData {
+				id: 1,
+				send: Box::new(send),
+				notif: Box::new(notif),
+				req: Box::new(req),
+				responses: HashMap::new(),
+			})),
+			logger,
+		)
+	}
+
+	/// Can be called from a polling thread or dispatched in any other way
+	pub async fn receive(&self, message: &[u8]) {
+		let mut g = self.0.lock().await;
+		let (id, payload) = get_id(message);
+		if id == 0 {
+			let mut notif_cb = clone_box(&*g.notif);
+			mem::drop(g);
+			let notif_val = <T::In as Channel>::Notif::decode(Pin::new(&mut &payload[..])).await;
+			notif_cb(notif_val, self.clone()).await
+		} else if 0 < id.bitand(1 << 63) {
+			let mut sender = g.responses.remove(&!id).expect("Received response for invalid message");
+			let _ = sender.send(message.to_vec()).await;
+		} else {
+			let message = <T::In as Channel>::Req::decode(Pin::new(&mut &payload[..])).await;
+			let mut req_cb = clone_box(&*g.req);
+			mem::drop(g);
+			let rn = self.clone();
+			req_cb(RequestHandle::new(rn, id), message).await;
+		}
+	}
+
+	pub async fn notify<N: Coding + Into<<T::Out as Channel>::Notif>>(&self, notif: N) {
+		let mut send = clone_box(&*self.0.lock().await.send);
+		let mut buf = vec![0; 8];
+		let msg: <T::Out as Channel>::Notif = notif.into();
+		msg.encode(Pin::new(&mut buf)).await;
+		send(&buf, self.clone()).await
+	}
+}
+
+pub trait DynRequester {
+	type Transfer;
+	fn logger(&self) -> &Logger;
+	/// Encode and send a request, then receive the response buffer.
+	fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply>;
+}
+
+pub struct MappedRequester<'a, T: 'a>(Box<dyn Fn(T) -> LocalBoxFuture<'a, RawReply> + 'a>, Logger);
+impl<'a, T> MappedRequester<'a, T> {
+	fn new<U: DynRequester + 'a, F: Fn(T) -> U::Transfer + 'a>(
+		req: U,
+		cb: F,
+		logger: Logger,
+	) -> Self {
+		let req_arc = Arc::new(req);
+		let cb_arc = Arc::new(cb);
+		MappedRequester(
+			Box::new(move |t| {
+				Box::pin(clone!(req_arc, cb_arc; async move { req_arc.raw_request(cb_arc(t)).await}))
+			}),
+			logger,
+		)
+	}
+}
+
+impl<T> DynRequester for MappedRequester<'_, T> {
+	type Transfer = T;
+	fn logger(&self) -> &Logger { &self.1 }
+	fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply> { self.0(data) }
+}
+
+impl<T: MsgSet> DynRequester for ReqNot<T> {
+	type Transfer = <T::Out as Channel>::Req;
+	fn logger(&self) -> &Logger { &self.1 }
+	fn raw_request(&self, req: Self::Transfer) -> LocalBoxFuture<'_, RawReply> {
 		Box::pin(async move {
-			self.o.flush().await?;
+			let mut g = self.0.lock().await;
-			self.drop_g.defuse();
+			let id = g.id;
-			Ok(())
+			g.id += 1;
+			let mut buf = id.to_be_bytes().to_vec();
+			req.encode(Pin::new(&mut buf)).await;
+			let (send, mut recv) = mpsc::channel(1);
+			g.responses.insert(id, send);
+			let mut send = clone_box(&*g.send);
+			mem::drop(g);
+			let rn = self.clone();
+			send(&buf, rn).await;
+			let items = recv.next().await;
+			RawReply(items.unwrap())
 		})
 	}
 }
 
-pub struct CommCtx {
+pub trait Requester: DynRequester {
-	exit: Sender<()>,
+	#[must_use = "These types are subject to change with protocol versions. \
-}
+    If you don't want to use the return value, At a minimum, force the type."]
-
+	fn request<R: Request + Into<Self::Transfer>>(
-impl CommCtx {
+		&self,
-	pub async fn exit(self) -> io::Result<()> {
+		data: R,
-		self.exit.clone().send(()).await.expect("quit channel dropped");
+	) -> impl Future<Output = R::Response>;
-		Ok(())
+	fn map<'a, U>(self, cb: impl Fn(U) -> Self::Transfer + 'a) -> MappedRequester<'a, U>
+	where Self: Sized + 'a {
+		let logger = self.logger().clone();
+		MappedRequester::new(self, cb, logger)
 	}
 }
 
-/// Establish bidirectional request-notification communication over a duplex
+impl<This: DynRequester + ?Sized> Requester for This {
-/// channel. The returned [IoClient] can be used for notifications immediately,
+	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
-/// but requests can only be received while the future is running. The future
+		let req = format!("{data:?}");
-/// will only resolve when [CommCtx::quit] is called. The generic type
+		let rep = R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await;
-/// parameters are associated with the client and serve to ensure with a runtime
+		let req_str = req.to_string();
-/// check that the correct message families are sent in the correct directions
+		if !req_str.starts_with("AtomPrint") && !req_str.starts_with("ExtAtomPrint") {
-/// across the channel.
+			writeln!(self.logger(), "Request {req} got response {rep:?}");
-pub fn io_comm(
-	o: Pin<Box<dyn AsyncWrite>>,
-	i: Pin<Box<dyn AsyncRead>>,
-) -> (impl Client + 'static, CommCtx, IoCommServer) {
-	let i = Rc::new(Mutex::new(i));
-	let o = Rc::new(Mutex::new(o));
-	let (onsub, client) = IoClient::new(o.clone());
-	let (exit, onexit) = channel(1);
-	(client, CommCtx { exit }, IoCommServer { o, i, onsub, onexit })
 		}
-pub struct IoCommServer {
+		rep
-	o: Rc<Mutex<Pin<Box<dyn AsyncWrite>>>>,
-	i: Rc<Mutex<Pin<Box<dyn AsyncRead>>>>,
-	onsub: Receiver<ReplySub>,
-	onexit: Receiver<()>,
-}
-impl IoCommServer {
-	pub async fn listen(
-		self,
-		notif: impl for<'a> AsyncFn(Box<dyn MsgReader<'a> + 'a>) -> io::Result<()>,
-		req: impl for<'a> AsyncFn(Box<dyn ReqReader<'a> + 'a>) -> io::Result<Receipt<'a>>,
-	) -> io::Result<()> {
-		let Self { o, i, onexit, onsub } = self;
-		enum Event {
-			Input(u64, IoGuard<dyn AsyncRead>),
-			Sub(ReplySub),
-			Exit,
-		}
-		let input_stream = try_stream(async |mut h| {
-			loop {
-				let mut g = Bound::async_new(i.clone(), async |i| i.lock().await).await;
-				match u64::decode(g.as_mut()).await {
-					Ok(id) => h.emit(Event::Input(id, g)).await,
-					Err(e) => match e.kind() {
-						io::ErrorKind::BrokenPipe
-						| io::ErrorKind::ConnectionAborted
-						| io::ErrorKind::UnexpectedEof => h.emit(Event::Exit).await,
-						_ => return Err(e),
-					},
 	}
 }
-		});
+
-		let (mut add_pending_req, fork_future) = LocalSet::new();
+impl<T: MsgSet> Clone for ReqNot<T> {
-		let mut fork_stream = pin!(fork_future.into_stream());
+	fn clone(&self) -> Self { Self(self.0.clone(), self.1.clone()) }
-		let mut pending_replies = HashMap::new();
-		'body: {
-			let mut shared = stream_select! {
-				pin!(input_stream) as Pin<&mut dyn Stream<Item = io::Result<Event>>>,
-				onsub.map(|sub| Ok(Event::Sub(sub))),
-				fork_stream.as_mut().map(|res| {
-					res.map(|()| panic!("this substream cannot exit while the loop is running") as Event)
-				}),
-				onexit.map(|()| Ok(Event::Exit)),
-			};
-			while let Some(next) = shared.next().await {
-				match next {
-					Err(e) => break 'body Err(e),
-					Ok(Event::Exit) => break,
-					Ok(Event::Sub(ReplySub { id, ack, cb })) => {
-						pending_replies.insert(id, cb);
-						// this is detected and logged on client
-						let _ = ack.send(());
-					},
-					Ok(Event::Input(0, read)) => {
-						let notif = &notif;
-						let notif_job =
-							async move { notif(Box::new(IoMsgReader { _pd: PhantomData, read })).await };
-						add_pending_req.send(Box::pin(notif_job)).await.unwrap();
-					},
-					// MSB == 0 is a request, !id where MSB == 1 is the corresponding response
-					Ok(Event::Input(id, read)) if (id & (1 << (u64::BITS - 1))) == 0 => {
-						let (o, req) = (o.clone(), &req);
-						let req_job = async move {
-							let mut prefix = Vec::new();
-							(!id).encode_vec(&mut prefix);
-							let _ = req(Box::new(IoReqReader { prefix: &pin!(prefix), read, write: &o })).await;
-							Ok(())
-						};
-						add_pending_req.send(Box::pin(req_job)).await.unwrap();
-					},
-					Ok(Event::Input(id, read)) => {
-						let cb = pending_replies.remove(&!id).expect("Reply to unrecognized request");
-						cb.send(read).unwrap_or_else(|_| panic!("Failed to send reply"));
-					},
-				}
-			}
-			Ok(())
-		}?;
-		mem::drop(add_pending_req);
-		while let Some(next) = fork_stream.next().await {
-			next?
-		}
-		let mut out = o.lock().await;
-		out.as_mut().flush().await?;
-		out.as_mut().close().await?;
-		Ok(())
-	}
 }
 
 #[cfg(test)]
 mod test {
-	use std::cell::RefCell;
+	use std::rc::Rc;
+	use std::sync::Arc;
 
-	use futures::channel::mpsc;
+	use futures::FutureExt;
-	use futures::{SinkExt, StreamExt, join};
+	use futures::lock::Mutex;
-	use orchid_api_derive::{Coding, Hierarchy};
+	use orchid_api_derive::Coding;
-	use orchid_api_traits::Request;
+	use orchid_api_traits::{Channel, Request};
-	use unsync_pipe::pipe;
+	use test_executors::spin_on;
 
-	use crate::future_debug::spin_on;
+	use super::{MsgSet, ReqNot};
-	use crate::reqnot::{ClientExt, MsgReaderExt, ReqReaderExt, io_comm};
+	use crate::logging::Logger;
+	use crate::reqnot::Requester as _;
+	use crate::{api, clone};
 
-	#[derive(Clone, Debug, PartialEq, Coding, Hierarchy)]
+	#[derive(Clone, Debug, Coding, PartialEq)]
-	#[extendable]
+	pub struct TestReq(u8);
-	struct TestNotif(u64);
+	impl Request for TestReq {
+		type Response = u8;
+	}
+
+	pub struct TestChan;
+	impl Channel for TestChan {
+		type Notif = u8;
+		type Req = TestReq;
+	}
+
+	pub struct TestMsgSet;
+	impl MsgSet for TestMsgSet {
+		type In = TestChan;
+		type Out = TestChan;
+	}
 
 	#[test]
 	fn notification() {
 		spin_on(async {
-			let (in1, out2) = pipe(1024);
+			let logger = Logger::new(api::LogStrategy::StdErr);
-			let (in2, out1) = pipe(1024);
+			let received = Arc::new(Mutex::new(None));
-			let (received, mut on_receive) = mpsc::channel(2);
+			let receiver = ReqNot::<TestMsgSet>::new(
-			let (_, recv_ctx, recv_srv) = io_comm(Box::pin(in2), Box::pin(out2));
+				logger.clone(),
-			let (sender, ..) = io_comm(Box::pin(in1), Box::pin(out1));
+				|_, _| panic!("Should not send anything"),
-			join!(
+				clone!(received; move |notif, _| clone!(received; async move {
-				async {
+					*received.lock().await = Some(notif);
-					recv_srv
+				}.boxed_local())),
-						.listen(
+				|_, _| panic!("Not receiving a request"),
-							async |notif| {
-								received.clone().send(notif.read::<TestNotif>().await?).await.unwrap();
-								Ok(())
-							},
-							async |_| panic!("Should receive notif, not request"),
-						)
-						.await
-						.unwrap()
-				},
-				async {
-					sender.notify(TestNotif(3)).await.unwrap();
-					assert_eq!(on_receive.next().await, Some(TestNotif(3)));
-					sender.notify(TestNotif(4)).await.unwrap();
-					assert_eq!(on_receive.next().await, Some(TestNotif(4)));
-					recv_ctx.exit().await.unwrap();
-				}
 			);
+			let sender = ReqNot::<TestMsgSet>::new(
+				logger,
+				clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
+					receiver.receive(d).await
+				}))),
+				|_, _| panic!("Should not receive notif"),
+				|_, _| panic!("Should not receive request"),
+			);
+			sender.notify(3).await;
+			assert_eq!(*received.lock().await, Some(3));
+			sender.notify(4).await;
+			assert_eq!(*received.lock().await, Some(4));
 		})
 	}
 
-	#[derive(Clone, Debug, Coding, Hierarchy)]
-	#[extendable]
-	struct DummyRequest(u64);
-	impl Request for DummyRequest {
-		type Response = u64;
-	}
-
 	#[test]
 	fn request() {
 		spin_on(async {
-			let (in1, out2) = pipe(1024);
+			let logger = Logger::new(api::LogStrategy::StdErr);
-			let (in2, out1) = pipe(1024);
+			let receiver = Rc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
-			let (_, srv_ctx, srv) = io_comm(Box::pin(in2), Box::pin(out2));
+			let sender = Rc::new(ReqNot::<TestMsgSet>::new(
-			let (client, client_ctx, client_srv) = io_comm(Box::pin(in1), Box::pin(out1));
+				logger.clone(),
-			join!(
+				clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
-				async {
+					receiver.lock().await.as_ref().unwrap().receive(d).await
-					srv
+				}))),
-						.listen(
+				|_, _| panic!("Should not receive notif"),
-							async |_| panic!("No notifs expected"),
+				|_, _| panic!("Should not receive request"),
-							async |mut req| {
+			));
-								let val = req.read_req::<DummyRequest>().await?;
+			*receiver.lock().await = Some(ReqNot::new(
-								req.reply(&val, &(val.0 + 1)).await
+				logger,
+				clone!(sender; move |d, _| clone!(sender; Box::pin(async move {
+					sender.receive(d).await
+				}))),
+				|_, _| panic!("Not receiving notifs"),
+				|hand, req| {
+					Box::pin(async move {
+						assert_eq!(req, TestReq(5));
+						hand.respond(&6u8).await
+					})
 				},
-						)
+			));
-						.await
+			let response = sender.request(TestReq(5)).await;
-						.unwrap()
-				},
-				async {
-					client_srv
-						.listen(
-							async |_| panic!("Not expecting ingress notif"),
-							async |_| panic!("Not expecting ingress req"),
-						)
-						.await
-						.unwrap()
-				},
-				async {
-					let response = client.request(DummyRequest(5)).await.unwrap();
 			assert_eq!(response, 6);
-					srv_ctx.exit().await.unwrap();
-					client_ctx.exit().await.unwrap();
-				}
-			);
 		})
 	}
-
-	#[test]
-	fn exit() {
-		spin_on(async {
-			let (input1, output1) = pipe(1024);
-			let (input2, output2) = pipe(1024);
-			let (reply_client, reply_context, reply_server) =
-				io_comm(Box::pin(input1), Box::pin(output2));
-			let (req_client, req_context, req_server) = io_comm(Box::pin(input2), Box::pin(output1));
-			let reply_context = RefCell::new(Some(reply_context));
-			let (exit, onexit) = futures::channel::oneshot::channel::<()>();
-			join!(
-				async move {
-					reply_server
-						.listen(
-							async |hand| {
-								let _notif = hand.read::<TestNotif>().await.unwrap();
-								let context = reply_context.borrow_mut().take().unwrap();
-								context.exit().await?;
-								Ok(())
-							},
-							async |mut hand| {
-								let req = hand.read_req::<DummyRequest>().await?;
-								hand.reply(&req, &(req.0 + 1)).await
-							},
-						)
-						.await
-						.unwrap();
-					exit.send(()).unwrap();
-					let _client = reply_client;
-				},
-				async move {
-					req_server
-						.listen(
-							async |_| panic!("Only the other server expected notifs"),
-							async |_| panic!("Only the other server expected requests"),
-						)
-						.await
-						.unwrap();
-					let _ctx = req_context;
-				},
-				async move {
-					req_client.request(DummyRequest(0)).await.unwrap();
-					req_client.notify(TestNotif(0)).await.unwrap();
-					onexit.await.unwrap();
-				}
-			)
-		});
-	}
 }

orchid-base/src/stash.rs

@@ -1,44 +0,0 @@
-//! A pattern for running async code from sync destructors and other
-//! unfortunately sync callbacks
-//!
-//! We create a task_local vecdeque which is moved into a thread_local whenever
-//! the task is being polled. A call to [stash] pushes the future onto this
-//! deque. Before [with_stash] returns, it pops everything from the deque
-//! individually and awaits each of them, pushing any additionally stashed
-//! futures onto the back of the same deque.
-
-use std::cell::RefCell;
-use std::collections::VecDeque;
-use std::pin::Pin;
-
-use task_local::task_local;
-
-#[derive(Default)]
-struct StashedFutures {
-	queue: RefCell<VecDeque<Pin<Box<dyn Future<Output = ()>>>>>,
-}
-
-task_local! {
-	static STASHED_FUTURES: StashedFutures;
-}
-
-/// Complete the argument future, and any futures spawned from it via [stash].
-/// This is useful mostly to guarantee that messaging destructors have run.
-pub async fn with_stash<F: Future>(fut: F) -> F::Output {
-	STASHED_FUTURES
-		.scope(StashedFutures::default(), async {
-			let val = fut.await;
-			while let Some(fut) = STASHED_FUTURES.with(|sf| sf.queue.borrow_mut().pop_front()) {
-				fut.await;
-			}
-			val
-		})
-		.await
-}
-
-/// Schedule a future to be run before the next [with_stash] guard ends. This is
-/// most useful for sending messages from destructors.
-pub fn stash<F: Future<Output = ()> + 'static>(fut: F) {
-	(STASHED_FUTURES.try_with(|sf| sf.queue.borrow_mut().push_back(Box::pin(fut))))
-		.expect("No stash! Timely completion cannot be guaranteed")
-}

orchid-base/src/tl_cache.rs

@@ -6,18 +6,4 @@ macro_rules! tl_cache {
 		}
 		V.with(|v| v.clone())
 	}};
-	(async $ty:ty : $expr:expr) => {{
-		type CellType = std::cell::OnceCell<$ty>;
-		thread_local! {
-			static V: CellType = std::cell::OnceCell::default();
-		}
-		match V.with(|cell: &CellType| cell.get().cloned()) {
-			Some(val) => val as $ty,
-			None => {
-				let val = $expr;
-				let _ = V.with(|cell: &CellType| cell.set(val.clone()));
-				val as $ty
-			},
-		}
-	}};
 }

orchid-base/src/tree.rs

@@ -14,7 +14,7 @@ use trait_set::trait_set;
 
 use crate::error::OrcErrv;
 use crate::format::{FmtCtx, FmtUnit, Format, Variants};
-use crate::interner::{IStr, es};
+use crate::interner::{Interner, Tok};
 use crate::location::{Pos, SrcRange};
 use crate::name::{Sym, VName, VPath};
 use crate::parse::Snippet;
@@ -28,6 +28,7 @@ pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::
 		api: &ApiEquiv,
 		ctx: &mut Self::FromApiCtx<'_>,
 		pos: SrcRange,
+		i: &Interner,
 	) -> impl Future<Output = Self>;
 	#[must_use]
 	fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>;
@@ -35,7 +36,7 @@ pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::
 impl<T: Clone + Debug + Coding> TokenVariant<T> for Never {
 	type FromApiCtx<'a> = ();
 	type ToApiCtx<'a> = ();
-	async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+	async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange, _: &Interner) -> Self {
 		panic!("Cannot deserialize Never")
 	}
 	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} }
@@ -107,19 +108,20 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 		hctx: &mut H::FromApiCtx<'_>,
 		xctx: &mut X::FromApiCtx<'_>,
 		src: &Sym,
+		i: &Interner,
 	) -> Self {
 		let pos = SrcRange::new(tt.range.clone(), src);
 		let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> {
 			BR,
-			NS(n => es(*n).await,
+			NS(n => Tok::from_api(*n, i).await,
-				b => Box::new(Self::from_api(b, hctx, xctx, src).boxed_local().await)),
+				b => Box::new(Self::from_api(b, hctx, xctx, src, i).boxed_local().await)),
-			Bottom(e => OrcErrv::from_api(e).await),
+			Bottom(e => OrcErrv::from_api(e, i).await),
-			LambdaHead(arg => Box::new(Self::from_api(arg, hctx, xctx, src).boxed_local().await)),
+			LambdaHead(arg => Box::new(Self::from_api(arg, hctx, xctx, src, i).boxed_local().await)),
-			Name(n => es(*n).await),
+			Name(n => Tok::from_api(*n, i).await),
-			S(*par, b => ttv_from_api(b, hctx, xctx, src).await),
+			S(*par, b => ttv_from_api(b, hctx, xctx, src, i).await),
-			Comment(c => es(*c).await),
+			Comment(c.clone()),
-			NewExpr(expr => X::from_api(expr, xctx, pos.clone()).await),
+			NewExpr(expr => X::from_api(expr, xctx, pos.clone(), i).await),
-			Handle(tk => H::from_api(tk, hctx, pos.clone()).await)
+			Handle(tk => H::from_api(tk, hctx, pos.clone(), i).await)
 		});
 		Self { sr: pos, tok }
 	}
@@ -133,7 +135,7 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 			BR,
 			NS(n.to_api(), b => Box::new(b.into_api(hctx, xctx).boxed_local().await)),
 			Bottom(e.to_api()),
-			Comment(c.to_api()),
+			Comment(c.clone()),
 			LambdaHead(arg => Box::new(arg.into_api(hctx, xctx).boxed_local().await)),
 			Name(nn.to_api()),
 			S(p, b => ttv_into_api(b, hctx, xctx).boxed_local().await),
@@ -143,8 +145,8 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 		api::TokenTree { range: self.sr.range.clone(), token }
 	}
 
-	pub fn is_kw(&self, tk: IStr) -> bool { self.tok.is_kw(tk) }
+	pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
-	pub fn as_name(&self) -> Option<IStr> {
+	pub fn as_name(&self) -> Option<Tok<String>> {
 		if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
 	}
 	pub fn as_multiname(&self) -> Result<VName, &TokTree<H, X>> {
@@ -191,10 +193,11 @@ pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
 	hctx: &mut H::FromApiCtx<'_>,
 	xctx: &mut X::FromApiCtx<'_>,
 	src: &Sym,
+	i: &Interner,
 ) -> Vec<TokTree<H, X>> {
 	stream(async |mut cx| {
 		for tok in tokv {
-			cx.emit(TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src).boxed_local().await).await
+			cx.emit(TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await).await
 		}
 	})
 	.collect()
@@ -237,14 +240,14 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
 	/// Information about the code addressed to the human reader or dev tooling
 	/// It has no effect on the behaviour of the program unless it's explicitly
 	/// read via reflection
-	Comment(IStr),
+	Comment(Rc<String>),
 	/// The part of a lambda between `\` and `.` enclosing the argument. The body
 	/// stretches to the end of the enclosing parens or the end of the const line
 	LambdaHead(Box<TokTree<H, X>>),
 	/// A binding, operator, or a segment of a namespaced::name
-	Name(IStr),
+	Name(Tok<String>),
 	/// A namespace prefix, like `my_ns::` followed by a token
-	NS(IStr, Box<TokTree<H, X>>),
+	NS(Tok<String>, Box<TokTree<H, X>>),
 	/// A line break
 	BR,
 	/// `()`, `[]`, or `{}`
@@ -260,7 +263,7 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
 }
 impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
 	pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
-	pub fn is_kw(&self, tk: IStr) -> bool { matches!(self, Token::Name(n) if *n == tk) }
+	pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) }
 	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
 		match self {
 			Self::S(p, b) if *p == par => Some(b),

orchid-base/src/virt_fs/common.rs

@@ -13,13 +13,13 @@ pub enum Loaded {
   Code(Arc<String>),
   /// Conceptually equivalent to the list of *.orc files in a folder, without
   /// the extension
-	Collection(Arc<Vec<IStr>>),
+  Collection(Arc<Vec<Tok<String>>>),
 }
 impl Loaded {
   /// Is the loaded item source code (not a collection)?
   pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
   /// Collect the elements in a collection rreport
-	pub fn collection(items: impl IntoIterator<Item = IStr>) -> Self {
+  pub fn collection(items: impl IntoIterator<Item = Tok<String>>) -> Self {
     Self::Collection(Arc::new(items.into_iter().collect()))
   }
 }
@@ -55,7 +55,7 @@ impl ErrorSansOrigin for CodeNotFound {
 /// formats and other sources for libraries and dependencies.
 pub trait VirtFS {
   /// Implementation of [VirtFS::read]
-	fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult;
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult;
   /// Discover information about a path without reading it.
   ///
   /// Implement this if your vfs backend can do expensive operations
@@ -68,7 +68,7 @@ pub trait VirtFS {
   }
   /// Convert a path into a human-readable string that is meaningful in the
   /// target context.
-	fn display(&self, path: &[IStr]) -> Option<String>;
+  fn display(&self, path: &[Tok<String>]) -> Option<String>;
   /// Convert the FS handler into a type-erased version of itself for packing in
   /// a tree.
   fn rc(self) -> Rc<dyn VirtFS>
@@ -81,11 +81,15 @@ pub trait VirtFS {
 }
 
 impl VirtFS for &dyn VirtFS {
-	fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult { (*self).get(path, full_path) }
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
-	fn display(&self, path: &[IStr]) -> Option<String> { (*self).display(path) }
+    (*self).get(path, full_path)
+  }
+  fn display(&self, path: &[Tok<String>]) -> Option<String> { (*self).display(path) }
 }
 
 impl<T: VirtFS + ?Sized> VirtFS for Rc<T> {
-	fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult { (**self).get(path, full_path) }
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
-	fn display(&self, path: &[IStr]) -> Option<String> { (**self).display(path) }
+    (**self).get(path, full_path)
+  }
+  fn display(&self, path: &[Tok<String>]) -> Option<String> { (**self).display(path) }
 }

orchid-base/src/virt_fs/decl.rs

@@ -32,7 +32,7 @@ impl<'a> Combine for &'a dyn VirtFS {
 pub type DeclTree = ModEntry<Rc<dyn VirtFS>, (), ()>;
 
 impl VirtFS for DeclTree {
-  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     match &self.member {
       ModMember::Item(it) => it.get(path, full_path),
       ModMember::Sub(module) => match path.split_first() {
@@ -44,7 +44,7 @@ impl VirtFS for DeclTree {
     }
   }
 
-  fn display(&self, path: &[IStr]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     let (head, tail) = path.split_first()?;
     match &self.member {
       ModMember::Item(it) => it.display(path),
@@ -54,16 +54,16 @@ impl VirtFS for DeclTree {
 }
 
 impl VirtFS for String {
-  fn display(&self, _: &[IStr]) -> Option<String> { None }
+  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
-  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.as_str().to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }
 }
 
 impl<'a> VirtFS for &'a str {
-  fn display(&self, _: &[IStr]) -> Option<String> { None }
+  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
-  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }

orchid-base/src/virt_fs/dir.rs

@@ -99,14 +99,14 @@ impl DirNode {
     }
   }
 
-  fn mk_pathbuf(&self, path: &[IStr]) -> PathBuf {
+  fn mk_pathbuf(&self, path: &[Tok<String>]) -> PathBuf {
     let mut fpath = self.root.clone();
     path.iter().for_each(|seg| fpath.push(seg.as_str()));
     fpath
   }
 }
 impl VirtFS for DirNode {
-  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     let fpath = self.mk_pathbuf(path);
     let mut binding = self.cached.borrow_mut();
     let (_, res) = (binding.raw_entry_mut().from_key(&fpath))
@@ -114,7 +114,7 @@ impl VirtFS for DirNode {
     res.clone()
   }
 
-  fn display(&self, path: &[IStr]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     let pathbuf = self.mk_pathbuf(path).with_extension(self.ext());
     Some(pathbuf.to_string_lossy().to_string())
   }

orchid-base/src/virt_fs/embed.rs

@@ -56,7 +56,7 @@ impl EmbeddedFS {
 }
 
 impl VirtFS for EmbeddedFS {
-  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     if path.is_empty() {
       return Ok(Loaded::collection(self.tree.keys(|_| true)));
     }
@@ -67,7 +67,7 @@ impl VirtFS for EmbeddedFS {
       ModMember::Sub(sub) => Loaded::collection(sub.keys(|_| true)),
     })
   }
-  fn display(&self, path: &[IStr]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     let Self { gen, suffix, .. } = self;
     Some(format!("{}{suffix} in {gen}", path.iter().join("/")))
   }

orchid-base/src/virt_fs/prefix.rs

@@ -21,18 +21,18 @@ impl<'a> PrefixFS<'a> {
       add: VPath::parse(add.as_ref()),
     }
   }
-  fn proc_path(&self, path: &[IStr]) -> Option<Vec<IStr>> {
+  fn proc_path(&self, path: &[Tok<String>]) -> Option<Vec<Tok<String>>> {
     let path = path.strip_prefix(self.remove.as_slice())?;
     Some(self.add.0.iter().chain(path).cloned().collect_vec())
   }
 }
 impl<'a> VirtFS for PrefixFS<'a> {
-  fn get(&self, path: &[IStr], full_path: &PathSlice) -> super::FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> super::FSResult {
     let path =
       self.proc_path(path).ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())?;
     self.wrapped.get(&path, full_path)
   }
-  fn display(&self, path: &[IStr]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     self.wrapped.display(&self.proc_path(path)?)
   }
 }

orchid-extension/Cargo.toml

@@ -8,18 +8,17 @@ edition = "2024"
 [dependencies]
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
-bound = "0.6.0"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.20"
-futures = { version = "0.3.31", default-features = false, features = [
+futures = { version = "0.3.31", features = [
 	"std",
 	"async-await",
-] }
+], default-features = false }
 futures-locks = "0.7.1"
-hashbrown = "0.16.1"
+hashbrown = "0.16.0"
 include_dir = { version = "0.7.4", optional = true }
 itertools = "0.14.0"
-konst = "0.4.3"
+konst = "0.4.2"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
 never = "0.1.0"
@@ -28,15 +27,15 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "5.1.0"
+ordered-float = "5.0.0"
-pastey = "0.2.1"
+pastey = "0.1.1"
+some_executor = "0.6.1"
 substack = "1.1.1"
 task-local = "0.1.0"
-tokio = { version = "1.49.0", optional = true, features = [] }
+tokio = { version = "1.47.1", optional = true, features = [] }
-tokio-util = { version = "0.7.17", optional = true, features = ["compat"] }
+tokio-util = { version = "0.7.16", optional = true, features = ["compat"] }
 
 trait-set = "0.3.0"
-unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
 
 [features]
 tokio = ["dep:tokio", "dep:tokio-util"]

orchid-extension/src/atom.rs

@@ -1,9 +1,7 @@
 use std::any::{Any, TypeId, type_name};
 use std::collections::HashMap;
-use std::fmt::{self, Debug};
+use std::fmt;
 use std::future::Future;
-use std::io;
-use std::marker::PhantomData;
 use std::num::NonZeroU32;
 use std::ops::Deref;
 use std::pin::Pin;
@@ -11,26 +9,25 @@ use std::rc::Rc;
 
 use dyn_clone::{DynClone, clone_box};
 use futures::future::LocalBoxFuture;
-use futures::{AsyncWrite, FutureExt, StreamExt, stream};
+use futures::{AsyncRead, AsyncWrite, FutureExt, StreamExt, stream};
 use orchid_api_derive::Coding;
-use orchid_api_traits::{Coding, Decode, InHierarchy, Request, UnderRoot, enc_vec};
+use orchid_api_traits::{Coding, Decode, Encode, Request, enc_vec};
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv, mk_errv_floating};
-use orchid_base::format::{FmtCtx, FmtUnit, Format, fmt, take_first};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, fmt};
-use orchid_base::interner::is;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::{Receipt, ReqHandle, ReqReader, ReqReaderExt};
+use orchid_base::reqnot::Requester;
 use trait_set::trait_set;
 
 use crate::api;
+use crate::context::{ctx, i};
 use crate::conv::ToExpr;
-use crate::entrypoint::request;
 // use crate::error::{ProjectError, ProjectResult};
 use crate::expr::{Expr, ExprData, ExprHandle, ExprKind};
 use crate::gen_expr::GExpr;
-use crate::system::{DynSystemCard, atom_by_idx, atom_info_for, cted, downcast_atom};
+use crate::system::{DynSystemCard, atom_info_for, downcast_atom};
 
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct AtomTypeId(pub NonZeroU32);
 
 pub trait AtomCard: 'static + Sized {
@@ -101,17 +98,14 @@ impl ForeignAtom {
 	pub(crate) fn new(handle: Rc<ExprHandle>, atom: api::Atom, pos: Pos) -> Self {
 		ForeignAtom { atom, expr: handle, pos }
 	}
-	pub async fn request<R: Request + UnderRoot<Root: AtomMethod>>(
+	pub async fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
-		&self,
+		let rep = (ctx().reqnot().request(api::Fwd(
-		r: R,
-	) -> Option<R::Response> {
-		let rep = (request(api::Fwd(
 			self.atom.clone(),
-			Sym::parse(<R as UnderRoot>::Root::NAME).await.unwrap().tok().to_api(),
+			Sym::parse(M::NAME, &i()).await.unwrap().tok().to_api(),
-			enc_vec(&r.into_root()),
+			enc_vec(&m).await,
 		)))
 		.await?;
-		Some(R::Response::decode_slice(&mut &rep[..]))
+		Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
 	}
 	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TAtom<T>, NotTypAtom> {
 		TAtom::downcast(self.ex().handle()).await
@@ -125,14 +119,10 @@ impl fmt::Debug for ForeignAtom {
 }
 impl Format for ForeignAtom {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::from_api(&request(api::ExtAtomPrint(self.atom.clone())).await)
+		FmtUnit::from_api(&ctx().reqnot().request(api::ExtAtomPrint(self.atom.clone())).await)
 	}
 }
 impl ToExpr for ForeignAtom {
-	async fn to_expr(self) -> Expr
-	where Self: Sized {
-		self.ex()
-	}
 	async fn to_gen(self) -> GExpr { self.ex().to_gen().await }
 }
 
@@ -144,56 +134,30 @@ pub struct NotTypAtom {
 impl NotTypAtom {
 	pub async fn mk_err(&self) -> OrcErrv {
 		mk_errv(
-			is("Not the expected type").await,
+			i().i("Not the expected type").await,
-			format!("The expression {} is not a {}", fmt(&self.expr).await, self.typ.name()),
+			format!("The expression {} is not a {}", fmt(&self.expr, &i()).await, self.typ.name()),
 			[self.pos.clone()],
 		)
 	}
 }
-impl Debug for NotTypAtom {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		f.debug_struct("NotTypAtom")
-			.field("pos", &self.pos)
-			.field("expr", &self.expr)
-			.field("typ.name", &self.typ.name())
-			.finish_non_exhaustive()
-	}
-}
 
-pub trait AtomMethod: Coding + InHierarchy {
+pub trait AtomMethod: Request + Coding {
 	const NAME: &str;
 }
 pub trait Supports<M: AtomMethod>: AtomCard {
-	fn handle<'a>(
+	fn handle(&self, req: M) -> impl Future<Output = <M as Request>::Response>;
-		&self,
-		hand: Box<dyn ReqHandle<'a> + '_>,
-		req: M,
-	) -> impl Future<Output = io::Result<Receipt<'a>>>;
 }
 
-trait HandleAtomMethod<A> {
+trait_set! {
-	fn handle<'a, 'b: 'a>(
+	trait AtomReqCb<A> = for<'a> Fn(
-		&'a self,
+		&'a A,
-		atom: &'a A,
+		Pin<&'a mut dyn AsyncRead>,
-		req: Box<dyn ReqReader<'b> + 'a>,
+		Pin<&'a mut dyn AsyncWrite>,
-	) -> LocalBoxFuture<'a, ()>;
+	) -> LocalBoxFuture<'a, ()>
-}
-struct AtomMethodHandler<M, A>(PhantomData<M>, PhantomData<A>);
-impl<M: AtomMethod, A: Supports<M>> HandleAtomMethod<A> for AtomMethodHandler<M, A> {
-	fn handle<'a, 'b: 'a>(
-		&'a self,
-		a: &'a A,
-		mut reader: Box<dyn ReqReader<'b> + 'a>,
-	) -> LocalBoxFuture<'a, ()> {
-		Box::pin(async {
-			let req = reader.read_req::<M>().await.unwrap();
-			let _ = Supports::<M>::handle(a, reader.finish().await, req).await.unwrap();
-		})
-	}
 }
 
 pub struct MethodSetBuilder<A: AtomCard> {
-	handlers: Vec<(&'static str, Rc<dyn HandleAtomMethod<A>>)>,
+	handlers: Vec<(&'static str, Rc<dyn AtomReqCb<A>>)>,
 }
 impl<A: AtomCard> MethodSetBuilder<A> {
 	pub fn new() -> Self { Self { handlers: vec![] } }
@@ -201,14 +165,20 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 	pub fn handle<M: AtomMethod>(mut self) -> Self
 	where A: Supports<M> {
 		assert!(!M::NAME.is_empty(), "AtomMethod::NAME cannoot be empty");
-		self.handlers.push((M::NAME, Rc::new(AtomMethodHandler::<M, A>(PhantomData, PhantomData))));
+		self.handlers.push((
+			M::NAME,
+			Rc::new(move |a: &A, req: Pin<&mut dyn AsyncRead>, rep: Pin<&mut dyn AsyncWrite>| {
+				async { Supports::<M>::handle(a, M::decode(req).await).await.encode(rep).await }
+					.boxed_local()
+			}),
+		));
 		self
 	}
 
 	pub async fn pack(&self) -> MethodSet<A> {
 		MethodSet {
 			handlers: stream::iter(self.handlers.iter())
-				.then(async |(k, v)| (Sym::parse(k).await.unwrap(), v.clone()))
+				.then(async |(k, v)| (Sym::parse(k, &i()).await.unwrap(), v.clone()))
 				.collect()
 				.await,
 		}
@@ -216,19 +186,20 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 }
 
 pub struct MethodSet<A: AtomCard> {
-	handlers: HashMap<Sym, Rc<dyn HandleAtomMethod<A>>>,
+	handlers: HashMap<Sym, Rc<dyn AtomReqCb<A>>>,
 }
 impl<A: AtomCard> MethodSet<A> {
 	pub(crate) async fn dispatch<'a>(
-		&self,
+		&'a self,
-		atom: &'_ A,
+		atom: &'a A,
 		key: Sym,
-		req: Box<dyn ReqReader<'a> + 'a>,
+		req: Pin<&'a mut dyn AsyncRead>,
+		rep: Pin<&'a mut dyn AsyncWrite>,
 	) -> bool {
 		match self.handlers.get(&key) {
 			None => false,
 			Some(handler) => {
-				handler.handle(atom, req).await;
+				handler(atom, req, rep).await;
 				true
 			},
 		}
@@ -257,17 +228,18 @@ impl<A: AtomicFeatures> TAtom<A> {
 			},
 		}
 	}
-	pub async fn request<R: Request + UnderRoot<Root: AtomMethod>>(&self, req: R) -> R::Response
+	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
-	where A: Supports<<R as UnderRoot>::Root> {
+	where A: Supports<M> {
-		R::Response::decode_slice(
+		M::Response::decode(Pin::new(
-			&mut &(request(api::Fwd(
+			&mut &(ctx().reqnot().request(api::Fwd(
 				self.untyped.atom.clone(),
-				Sym::parse(<R as UnderRoot>::Root::NAME).await.unwrap().tok().to_api(),
+				Sym::parse(M::NAME, &i()).await.unwrap().tok().to_api(),
-				enc_vec(&req.into_root()),
+				enc_vec(&req).await,
 			)))
 			.await
 			.unwrap()[..],
-		)
+		))
+		.await
 	}
 }
 impl<A: AtomicFeatures> Deref for TAtom<A> {
@@ -292,11 +264,12 @@ pub trait AtomDynfo: 'static {
 	fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
 	fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
 	fn print<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit>;
-	fn handle_req<'a>(
+	fn handle_req<'a, 'b: 'a, 'c: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
 		key: Sym,
-		req: Box<dyn ReqReader<'a> + 'a>,
+		req: Pin<&'b mut dyn AsyncRead>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool>;
 	fn command<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>>;
 	fn serialize<'a, 'b: 'a>(
@@ -333,25 +306,10 @@ impl Format for AtomFactory {
 	}
 }
 
-pub async fn err_not_callable(unit: &FmtUnit) -> OrcErrv {
+pub async fn err_not_callable() -> OrcErrv {
-	mk_errv_floating(
+	mk_errv_floating(i().i("This atom is not callable").await, "Attempted to apply value as function")
-		is("This atom is not callable").await,
-		format!("Attempted to apply {} as function", take_first(unit, false)),
-	)
 }
 
-pub async fn err_not_command(unit: &FmtUnit) -> OrcErrv {
+pub async fn err_not_command() -> OrcErrv {
-	mk_errv_floating(
+	mk_errv_floating(i().i("This atom is not a command").await, "Settled on an inactionable value")
-		is("This atom is not a command").await,
-		format!("Settled on {} which is an inactionable value", take_first(unit, false)),
-	)
-}
-
-/// Read the type ID prefix from an atom, return type information and the rest
-/// of the data
-pub(crate) fn resolve_atom_type(atom: &api::Atom) -> (Box<dyn AtomDynfo>, AtomTypeId, &[u8]) {
-	let mut data = &atom.data.0[..];
-	let tid = AtomTypeId::decode_slice(&mut data);
-	let atom_record = atom_by_idx(cted().inst().card(), tid).expect("Unrecognized atom type ID");
-	(atom_record, tid, data)
 }

orchid-extension/src/atom_owned.rs

@@ -1,53 +1,50 @@
 use std::any::{Any, TypeId, type_name};
 use std::borrow::Cow;
-use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
 use std::num::NonZero;
 use std::ops::Deref;
 use std::pin::Pin;
-use std::rc::Rc;
+use std::sync::atomic::AtomicU64;
 
 use async_once_cell::OnceCell;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::{LocalBoxFuture, ready};
-use futures::{AsyncWrite, FutureExt};
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use futures_locks::{RwLock, RwLockReadGuard};
 use itertools::Itertools;
 use memo_map::MemoMap;
 use never::Never;
 use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
-use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, Format, take_first};
+use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, take_first};
-use orchid_base::logging::log;
 use orchid_base::name::Sym;
-use task_local::task_local;
 
 use crate::api;
 use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
 	MethodSetBuilder, TAtom, err_not_callable, err_not_command, get_info,
 };
+use crate::context::{SysCtxEntry, ctx, i};
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
-use crate::system::{cted, sys_id};
+use crate::system_ctor::CtedObj;
 
 pub struct OwnedVariant;
 impl AtomicVariant for OwnedVariant {}
 impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVariant> for A {
 	fn _factory(self) -> AtomFactory {
 		AtomFactory::new(async move || {
-			let obj_store = get_obj_store();
+			let serial = ctx()
-			let atom_id = {
+				.get_or_default::<ObjStore>()
-				let mut id = obj_store.next_id.borrow_mut();
+				.next_id
-				*id += 1;
+				.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-				api::AtomId(NonZero::new(*id + 1).unwrap())
+			let atom_id = api::AtomId(NonZero::new(serial + 1).unwrap());
-			};
+			let (typ_id, _) = get_info::<A>(ctx().get::<CtedObj>().inst().card());
-			let (typ_id, _) = get_info::<A>(cted().inst().card());
+			let mut data = enc_vec(&typ_id).await;
-			let mut data = enc_vec(&typ_id);
 			self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
-			obj_store.objects.read().await.insert(atom_id, Box::new(self));
+			ctx().get_or_default::<ObjStore>().objects.read().await.insert(atom_id, Box::new(self));
-			api::Atom { drop: Some(atom_id), data: api::AtomData(data), owner: sys_id() }
+			api::Atom { drop: Some(atom_id), data: api::AtomData(data), owner: ctx().sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { OwnedAtomDynfo { msbuild: A::reg_reqs(), ms: OnceCell::new() } }
@@ -62,7 +59,7 @@ pub(crate) struct AtomReadGuard<'a> {
 }
 impl<'a> AtomReadGuard<'a> {
 	async fn new(id: api::AtomId) -> Self {
-		let guard = get_obj_store().objects.read().await;
+		let guard = ctx().get_or_default::<ObjStore>().objects.read().await;
 		if guard.get(&id).is_none() {
 			panic!("Received invalid atom ID: {id:?}");
 		}
@@ -76,7 +73,7 @@ impl Deref for AtomReadGuard<'_> {
 
 /// Remove an atom from the store
 pub(crate) async fn take_atom(id: api::AtomId) -> Box<dyn DynOwnedAtom> {
-	let mut g = get_obj_store().objects.write().await;
+	let mut g = ctx().get_or_default::<ObjStore>().objects.write().await;
 	g.remove(&id).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0))
 }
 
@@ -89,46 +86,29 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(data, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
 		Box::pin(async {
-			Box::new(<T as AtomCard>::Data::decode_slice(&mut &data[..])) as Box<dyn Any>
+			Box::new(<T as AtomCard>::Data::decode(Pin::new(&mut &data[..])).await) as Box<dyn Any>
 		})
 	}
 	fn call(&self, AtomCtx(_, id): AtomCtx, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
-		Box::pin(async move {
+		Box::pin(async move { take_atom(id.unwrap()).await.dyn_call(arg).await })
-			writeln!(
-				log("msg"),
-				"owned call {} {}",
-				take_first(&AtomReadGuard::new(id.unwrap()).await.dyn_print().await, false),
-				take_first(&arg.print(&FmtCtxImpl::default()).await, true),
-			)
-			.await;
-			take_atom(id.unwrap()).await.dyn_call(arg).await
-		})
 	}
 	fn call_ref<'a>(&'a self, AtomCtx(_, id): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move {
+		Box::pin(async move { AtomReadGuard::new(id.unwrap()).await.dyn_call_ref(arg).await })
-			writeln!(
-				log("msg"),
-				"owned call_ref {} {}",
-				take_first(&AtomReadGuard::new(id.unwrap()).await.dyn_print().await, false),
-				take_first(&arg.print(&FmtCtxImpl::default()).await, true),
-			)
-			.await;
-			AtomReadGuard::new(id.unwrap()).await.dyn_call_ref(arg).await
-		})
 	}
 	fn print(&self, AtomCtx(_, id): AtomCtx<'_>) -> LocalBoxFuture<'_, FmtUnit> {
 		Box::pin(async move { AtomReadGuard::new(id.unwrap()).await.dyn_print().await })
 	}
-	fn handle_req<'a>(
+	fn handle_req<'a, 'b: 'a, 'c: 'a>(
 		&'a self,
-		AtomCtx(_, id): AtomCtx<'a>,
+		AtomCtx(_, id): AtomCtx,
 		key: Sym,
-		req: Box<dyn orchid_base::reqnot::ReqReader<'a> + 'a>,
+		req: Pin<&'b mut dyn AsyncRead>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let a = AtomReadGuard::new(id.unwrap()).await;
 			let ms = self.ms.get_or_init(self.msbuild.pack()).await;
-			ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), key, req).await
+			ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), key, req, rep).await
 		})
 	}
 	fn command<'a>(
@@ -147,7 +127,7 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async move {
 			let id = id.unwrap();
-			id.encode(write.as_mut()).await.unwrap();
+			id.encode(write.as_mut()).await;
 			AtomReadGuard::new(id).await.dyn_serialize(write).await
 		})
 	}
@@ -175,7 +155,7 @@ pub trait DeserializeCtx: Sized {
 
 struct DeserCtxImpl<'a>(&'a [u8]);
 impl DeserializeCtx for DeserCtxImpl<'_> {
-	async fn read<T: Decode>(&mut self) -> T { T::decode(Pin::new(&mut self.0)).await.unwrap() }
+	async fn read<T: Decode>(&mut self) -> T { T::decode(Pin::new(&mut self.0)).await }
 	fn is_empty(&self) -> bool { self.0.is_empty() }
 }
 
@@ -227,7 +207,7 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	fn val(&self) -> impl Future<Output = Cow<'_, Self::Data>>;
 	#[allow(unused_variables)]
 	fn call_ref(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot(err_not_callable(&self.dyn_print().await).await) }
+		async move { bot(err_not_callable().await) }
 	}
 	fn call(self, arg: Expr) -> impl Future<Output = GExpr> {
 		async {
@@ -238,7 +218,7 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	}
 	#[allow(unused_variables)]
 	fn command(self) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(&self.dyn_print().await).await) }
+		async move { Err(err_not_command().await) }
 	}
 	#[allow(unused_variables)]
 	fn free(self) -> impl Future<Output = ()> { async {} }
@@ -286,7 +266,7 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn as_any_ref(&self) -> &dyn Any { self }
 	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()> {
-		async { self.val().await.as_ref().encode(buffer).await.unwrap() }.boxed_local()
+		async { self.val().await.as_ref().encode(buffer).await }.boxed_local()
 	}
 	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
 		self.call_ref(arg).boxed_local()
@@ -299,7 +279,7 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	}
 	fn dyn_free(self: Box<Self>) -> LocalBoxFuture<'static, ()> { self.free().boxed_local() }
 	fn dyn_print(&self) -> LocalBoxFuture<'_, FmtUnit> {
-		async move { self.print_atom(&FmtCtxImpl::default()).await }.boxed_local()
+		async move { self.print_atom(&FmtCtxImpl { i: &i() }).await }.boxed_local()
 	}
 	fn dyn_serialize<'a>(
 		&'a self,
@@ -314,24 +294,13 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 
 #[derive(Default)]
 pub(crate) struct ObjStore {
-	pub(crate) next_id: RefCell<u64>,
+	pub(crate) next_id: AtomicU64,
 	pub(crate) objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
-
+impl SysCtxEntry for ObjStore {}
-task_local! {
-	static OBJ_STORE: Rc<ObjStore>;
-}
-
-pub(crate) fn with_obj_store<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
-	Box::pin(OBJ_STORE.scope(Rc::new(ObjStore::default()), fut))
-}
-
-pub(crate) fn get_obj_store() -> Rc<ObjStore> {
-	OBJ_STORE.try_with(|store| store.clone()).expect("Owned atom store not initialized")
-}
 
 pub async fn own<A: OwnedAtom>(typ: &TAtom<A>) -> A {
-	let g = get_obj_store().objects.read().await;
+	let g = ctx().get_or_default::<ObjStore>().objects.read().await;
 	let atom_id = typ.untyped.atom.drop.expect("Owned atoms always have a drop ID");
 	let dyn_atom =
 		g.get(&atom_id).expect("Atom ID invalid; atom type probably not owned by this crate");
@@ -339,7 +308,8 @@ pub async fn own<A: OwnedAtom>(typ: &TAtom<A>) -> A {
 }
 
 pub async fn debug_print_obj_store(show_atoms: bool) {
-	let store = get_obj_store();
+	let ctx = ctx();
+	let store = ctx.get_or_default::<ObjStore>();
 	let keys = store.objects.read().await.keys().cloned().collect_vec();
 	let mut message = "Atoms in store:".to_string();
 	if !show_atoms {
@@ -356,5 +326,5 @@ pub async fn debug_print_obj_store(show_atoms: bool) {
 			message += &format!("\n{k:?} -> {}", take_first(&atom.dyn_print().await, true));
 		}
 	}
-	writeln!(log("debug"), "{message}").await
+	eprintln!("{message}")
 }

orchid-extension/src/atom_thin.rs

@@ -4,11 +4,10 @@ use std::pin::Pin;
 
 use async_once_cell::OnceCell;
 use futures::future::LocalBoxFuture;
-use futures::{AsyncWrite, FutureExt};
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
-use orchid_base::logging::log;
 use orchid_base::name::Sym;
 
 use crate::api;
@@ -16,19 +15,20 @@ use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
 	MethodSetBuilder, err_not_callable, err_not_command, get_info,
 };
+use crate::context::ctx;
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
-use crate::system::{cted, sys_id};
+use crate::system_ctor::CtedObj;
 
 pub struct ThinVariant;
 impl AtomicVariant for ThinVariant {}
 impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant> for A {
 	fn _factory(self) -> AtomFactory {
 		AtomFactory::new(async move || {
-			let (id, _) = get_info::<A>(cted().inst().card());
+			let (id, _) = get_info::<A>(ctx().get::<CtedObj>().inst().card());
-			let mut buf = enc_vec(&id);
+			let mut buf = enc_vec(&id).await;
-			self.encode_vec(&mut buf);
+			self.encode(Pin::new(&mut buf)).await;
-			api::Atom { drop: None, data: api::AtomData(buf), owner: sys_id() }
+			api::Atom { drop: None, data: api::AtomData(buf), owner: ctx().sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { ThinAtomDynfo { msbuild: Self::reg_reqs(), ms: OnceCell::new() } }
@@ -41,35 +41,36 @@ pub struct ThinAtomDynfo<T: ThinAtom> {
 }
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn print<'a>(&self, AtomCtx(buf, _): AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit> {
-		Box::pin(async move { T::decode_slice(&mut &buf[..]).print().await })
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.print().await })
 	}
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
-		Box::pin(async { Box::new(T::decode_slice(&mut &buf[..])) as Box<dyn Any> })
+		Box::pin(async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> })
 	}
 	fn call<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move { T::decode_slice(&mut &buf[..]).call(arg).await })
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
 	}
 	fn call_ref<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move { T::decode_slice(&mut &buf[..]).call(arg).await })
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
 	}
-	fn handle_req<'a>(
+	fn handle_req<'a, 'm1: 'a, 'm2: 'a>(
 		&'a self,
-		AtomCtx(buf, ..): AtomCtx<'a>,
+		AtomCtx(buf, _): AtomCtx<'a>,
 		key: Sym,
-		req: Box<dyn orchid_base::reqnot::ReqReader<'a> + 'a>,
+		req: Pin<&'m1 mut dyn AsyncRead>,
+		rep: Pin<&'m2 mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack()).await;
-			ms.dispatch(&T::decode_slice(&mut &buf[..]), key, req).await
+			ms.dispatch(&T::decode(Pin::new(&mut &buf[..])).await, key, req, rep).await
 		})
 	}
 	fn command<'a>(
 		&'a self,
 		AtomCtx(buf, _): AtomCtx<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>> {
-		async move { T::decode_slice(&mut &buf[..]).command().await }.boxed_local()
+		async move { T::decode(Pin::new(&mut &buf[..])).await.command().await }.boxed_local()
 	}
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
@@ -77,18 +78,18 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 		write: Pin<&'b mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async {
-			T::decode_slice(&mut &ctx.0[..]).encode(write).await.unwrap();
+			T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await;
 			Some(Vec::new())
 		})
 	}
 	fn deserialize<'a>(&'a self, data: &'a [u8], refs: &'a [Expr]) -> LocalBoxFuture<'a, api::Atom> {
 		assert!(refs.is_empty(), "Refs found when deserializing thin atom");
-		Box::pin(async { T::decode_slice(&mut &data[..])._factory().build().await })
+		Box::pin(async { T::decode(Pin::new(&mut &data[..])).await._factory().build().await })
 	}
 	fn drop<'a>(&'a self, AtomCtx(buf, _): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> {
 		Box::pin(async move {
-			let string_self = T::decode_slice(&mut &buf[..]).print().await;
+			let string_self = T::decode(Pin::new(&mut &buf[..])).await.print().await;
-			writeln!(log("warn"), "Received drop signal for non-drop atom {string_self:?}").await;
+			writeln!(ctx().logger(), "Received drop signal for non-drop atom {string_self:?}");
 		})
 	}
 }
@@ -98,11 +99,11 @@ pub trait ThinAtom:
 {
 	#[allow(unused_variables)]
 	fn call(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot(err_not_callable(&self.print().await).await) }
+		async move { bot(err_not_callable().await) }
 	}
 	#[allow(unused_variables)]
 	fn command(&self) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(&self.print().await).await) }
+		async move { Err(err_not_command().await) }
 	}
 	#[allow(unused_variables)]
 	fn print(&self) -> impl Future<Output = FmtUnit> {

orchid-extension/src/binary.rs

@@ -1,53 +0,0 @@
-use std::rc::Rc;
-
-use futures::future::LocalBoxFuture;
-use orchid_base::binary::future_to_vt;
-
-use crate::api;
-use crate::entrypoint::ExtensionBuilder;
-use crate::ext_port::ExtPort;
-
-pub type ExtCx = api::binary::ExtensionContext;
-
-struct Spawner(api::binary::SpawnerBin);
-impl Drop for Spawner {
-	fn drop(&mut self) { (self.0.drop)(self.0.data) }
-}
-impl Spawner {
-	pub fn spawn(&self, fut: LocalBoxFuture<'static, ()>) {
-		(self.0.spawn)(self.0.data, future_to_vt(fut))
-	}
-}
-
-pub fn orchid_extension_main_body(cx: ExtCx, builder: ExtensionBuilder) {
-	let spawner = Spawner(cx.spawner);
-	builder.build(ExtPort {
-		input: Box::pin(cx.input),
-		output: Box::pin(cx.output),
-		log: Box::pin(cx.log),
-		spawn: Rc::new(move |fut| spawner.spawn(fut)),
-	});
-}
-
-/// Generate entrypoint for the dylib extension loader
-///
-/// # Usage
-///
-/// ```
-/// #[macro_use]
-/// use orchid_extension::dylib_main;
-/// use orchid_extension::entrypoint::ExtensionBuilder;
-///
-/// dylib_main! {
-///   ExtensionBuilder::new("orchid-std::main")
-/// }
-/// ```
-#[macro_export]
-macro_rules! dylib_main {
-	($builder:expr) => {
-		#[unsafe(no_mangle)]
-		pub extern "C" fn orchid_extension_main(cx: ::orchid_api::binary::ExtensionContext) {
-			$crate::binary::orchid_extension_main_body(cx, $builder);
-		}
-	};
-}

orchid-extension/src/context.rs

@@ -0,0 +1,90 @@
+use std::any::{Any, TypeId, type_name};
+use std::fmt;
+use std::num::NonZero;
+use std::rc::Rc;
+
+use memo_map::MemoMap;
+use orchid_base::builtin::Spawner;
+use orchid_base::interner::Interner;
+use orchid_base::logging::Logger;
+use orchid_base::reqnot::ReqNot;
+use task_local::task_local;
+
+use crate::api;
+use crate::system_ctor::CtedObj;
+
+#[derive(Clone)]
+pub struct SysCtx(Rc<MemoMap<TypeId, Box<dyn Any>>>);
+impl SysCtx {
+	pub fn new(
+		id: api::SysId,
+		i: Interner,
+		reqnot: ReqNot<api::ExtMsgSet>,
+		spawner: Spawner,
+		logger: Logger,
+		cted: CtedObj,
+	) -> Self {
+		let this = Self(Rc::new(MemoMap::new()));
+		this.add(id).add(i).add(reqnot).add(spawner).add(logger).add(cted);
+		this
+	}
+	pub fn add<T: SysCtxEntry>(&self, t: T) -> &Self {
+		assert!(self.0.insert(TypeId::of::<T>(), Box::new(t)), "Key already exists");
+		self
+	}
+	pub fn get_or_insert<T: SysCtxEntry>(&self, f: impl FnOnce() -> T) -> &T {
+		(self.0.get_or_insert_owned(TypeId::of::<T>(), || Box::new(f())).downcast_ref())
+			.expect("Keyed by TypeId")
+	}
+	pub fn get_or_default<T: SysCtxEntry + Default>(&self) -> &T { self.get_or_insert(T::default) }
+	pub fn try_get<T: SysCtxEntry>(&self) -> Option<&T> {
+		Some(self.0.get(&TypeId::of::<T>())?.downcast_ref().expect("Keyed by TypeId"))
+	}
+	pub fn get<T: SysCtxEntry>(&self) -> &T {
+		self.try_get().unwrap_or_else(|| panic!("Context {} missing", type_name::<T>()))
+	}
+	/// Shorthand to get the messaging link
+	pub fn reqnot(&self) -> &ReqNot<api::ExtMsgSet> { self.get::<ReqNot<api::ExtMsgSet>>() }
+	/// Shorthand to get the system ID
+	pub fn sys_id(&self) -> api::SysId { *self.get::<api::SysId>() }
+	/// Spawn a task that will eventually be executed asynchronously
+	pub fn spawn(&self, f: impl Future<Output = ()> + 'static) {
+		(self.get::<Spawner>())(Box::pin(CTX.scope(self.clone(), f)))
+	}
+	/// Shorthand to get the logger
+	pub fn logger(&self) -> &Logger { self.get::<Logger>() }
+	/// Shorthand to get the constructed system object
+	pub fn cted(&self) -> &CtedObj { self.get::<CtedObj>() }
+}
+impl fmt::Debug for SysCtx {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "SysCtx({:?})", self.sys_id())
+	}
+}
+pub trait SysCtxEntry: 'static + Sized {}
+impl SysCtxEntry for api::SysId {}
+impl SysCtxEntry for ReqNot<api::ExtMsgSet> {}
+impl SysCtxEntry for Spawner {}
+impl SysCtxEntry for CtedObj {}
+impl SysCtxEntry for Logger {}
+impl SysCtxEntry for Interner {}
+
+task_local! {
+	static CTX: SysCtx;
+}
+
+pub async fn with_ctx<F: Future>(ctx: SysCtx, f: F) -> F::Output { CTX.scope(ctx, f).await }
+pub fn ctx() -> SysCtx { CTX.get() }
+
+/// Shorthand to get the [Interner] instance
+pub fn i() -> Interner { ctx().get::<Interner>().clone() }
+
+pub fn mock_ctx() -> SysCtx {
+	let ctx = SysCtx(Rc::default());
+	ctx
+		.add(Logger::new(api::LogStrategy::StdErr))
+		.add(Interner::new_master())
+		.add::<Spawner>(Rc::new(|_| panic!("Cannot fork in test environment")))
+		.add(api::SysId(NonZero::<u16>::MIN));
+	ctx
+}

orchid-extension/src/conv.rs

@@ -4,14 +4,13 @@ use std::pin::Pin;
 use dyn_clone::DynClone;
 use never::Never;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
-use orchid_base::format::{Format, fmt};
-use orchid_base::interner::is;
 use orchid_base::location::Pos;
 use trait_set::trait_set;
 
-use crate::atom::{AtomicFeatures, ForeignAtom, TAtom};
+use crate::atom::{AtomicFeatures, ForeignAtom, TAtom, ToAtom};
-use crate::expr::{Expr, ExprKind};
+use crate::context::i;
-use crate::gen_expr::{GExpr, bot};
+use crate::expr::Expr;
+use crate::gen_expr::{GExpr, atom, bot};
 
 pub trait TryFromExpr: Sized {
 	fn try_from_expr(expr: Expr) -> impl Future<Output = OrcRes<Self>>;
@@ -27,17 +26,14 @@ impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
 	}
 }
 
-async fn err_not_atom(pos: Pos, value: &impl Format) -> OrcErrv {
+async fn err_not_atom(pos: Pos) -> OrcErrv {
-	mk_errv(is("Expected an atom").await, format!("{} is not an atom", fmt(value).await), [pos])
+	mk_errv(i().i("Expected an atom").await, "This expression is not an atom", [pos])
 }
 
 impl TryFromExpr for ForeignAtom {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
-		if let ExprKind::Bottom(err) = &expr.data().await.kind {
+		match expr.atom().await {
-			return Err(err.clone());
+			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone()).await),
-		}
-		match expr.clone().atom().await {
-			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &expr).await),
 			Ok(f) => Ok(f),
 		}
 	}
@@ -111,6 +107,10 @@ impl<T: ToExpr> ToExpr for OrcRes<T> {
 	}
 }
 
+impl<A: ToAtom> ToExpr for A {
+	async fn to_gen(self) -> GExpr { atom(self) }
+}
+
 impl ToExpr for Never {
 	async fn to_gen(self) -> GExpr { match self {} }
 }

orchid-extension/src/coroutine_exec.rs

@@ -13,7 +13,7 @@ use crate::atom::Atomic;
 use crate::atom_owned::{OwnedAtom, OwnedVariant};
 use crate::conv::{ToExpr, TryFromExpr};
 use crate::expr::Expr;
-use crate::gen_expr::{GExpr, arg, call, lambda, new_atom, seq};
+use crate::gen_expr::{GExpr, arg, call, lambda, seq};
 
 enum Command {
 	Execute(GExpr, Sender<Expr>),
@@ -34,11 +34,14 @@ impl BuilderCoroutine {
 			None => panic!("Before the stream ends, we should have gotten a Halt"),
 			Some(Command::Halt(expr)) => expr,
 			Some(Command::Execute(expr, reply)) => call(
-				lambda(0, [seq([arg(0)], call(new_atom(Replier { reply, builder: self }), [arg(0)]))]),
+				lambda(0, [seq(
+					[arg(0)],
+					call(Replier { reply, builder: self }.to_gen().await, [arg(0)]),
+				)]),
 				[expr],
 			),
 			Some(Command::Register(expr, reply)) =>
-				call(new_atom(Replier { reply, builder: self }), [expr]),
+				call(Replier { reply, builder: self }.to_gen().await, [expr]),
 		}
 	}
 }
@@ -64,8 +67,8 @@ impl OwnedAtom for Replier {
 
 pub async fn exec<R: ToExpr>(f: impl for<'a> AsyncFnOnce(ExecHandle<'a>) -> R + 'static) -> GExpr {
 	let (cmd_snd, cmd_recv) = channel(0);
-	let halt =
+	let halt = async { Command::Halt(f(ExecHandle(cmd_snd, PhantomData)).await.to_gen().await) }
-		async { Command::Halt(f(ExecHandle(cmd_snd, PhantomData)).await.to_gen().await) }.into_stream();
+		.into_stream();
 	let coro = BuilderCoroutine(Rc::new(BuilderCoroutineData {
 		receiver: Mutex::new(stream::select(halt, cmd_recv).boxed_local()),
 	}));

orchid-extension/src/entrypoint.rs

@@ -5,323 +5,349 @@ use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
 
+use futures::channel::mpsc::{Receiver, Sender, channel};
 use futures::future::{LocalBoxFuture, join_all};
-use futures::{AsyncWriteExt, StreamExt, stream};
+use futures::lock::Mutex;
+use futures::{FutureExt, SinkExt, StreamExt, stream, stream_select};
+use futures_locks::RwLock;
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api_traits::{Decode, Encode, Request, UnderRoot, enc_vec};
+use orchid_api_traits::{Decode, UnderRoot, enc_vec};
+use orchid_base::builtin::{ExtInit, ExtPort, Spawner};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
-use orchid_base::error::try_with_reporter;
+use orchid_base::clone;
-use orchid_base::interner::{es, is, with_interner};
+use orchid_base::error::Reporter;
-use orchid_base::logging::{log, with_logger};
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::logging::Logger;
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Snippet};
-use orchid_base::reqnot::{
+use orchid_base::reqnot::{ReqNot, RequestHandle, Requester};
-	Client, ClientExt, CommCtx, MsgReader, MsgReaderExt, ReqHandleExt, ReqReaderExt, Witness, io_comm,
-};
-use orchid_base::stash::with_stash;
 use orchid_base::tree::{TokenVariant, ttv_from_api};
 use substack::Substack;
-use task_local::task_local;
+use trait_set::trait_set;
 
 use crate::api;
-use crate::atom::{AtomCtx, AtomTypeId, resolve_atom_type};
+use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId};
-use crate::atom_owned::{take_atom, with_obj_store};
+use crate::atom_owned::take_atom;
+use crate::context::{SysCtx, ctx, i, with_ctx};
 use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
-use crate::ext_port::ExtPort;
-use crate::func_atom::with_funs_ctx;
-use crate::interner::new_interner;
 use crate::lexer::{LexContext, ekey_cascade, ekey_not_applicable};
-use crate::logger::LoggerImpl;
+use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api};
-use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api, with_parsed_const_ctx};
+use crate::system::atom_by_idx;
-use crate::reflection::with_refl_roots;
+use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::system::{SysCtx, atom_by_idx, cted, with_sys};
+use crate::tree::{LazyMemberFactory, TreeIntoApiCtxImpl};
-use crate::system_ctor::{CtedObj, DynSystemCtor, SystemCtor};
-use crate::tree::{TreeIntoApiCtxImpl, get_lazy, with_lazy_member_store};
-use crate::trivial_req::TrivialReqCycle;
 
-task_local::task_local! {
+pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
-	static CLIENT: Rc<dyn Client>;
+pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
-	static CTX: Rc<RefCell<Option<CommCtx>>>;
+
+pub struct ExtensionData {
+	pub name: &'static str,
+	pub systems: &'static [&'static dyn DynSystemCtor],
+}
+impl ExtensionData {
+	pub fn new(name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
+		Self { name, systems }
+	}
 }
 
-fn get_client() -> Rc<dyn Client> { CLIENT.get() }
+pub enum MemberRecord {
-pub async fn exit() {
+	Gen(Vec<Tok<String>>, LazyMemberFactory),
-	let cx = CTX.get().borrow_mut().take();
+	Res,
-	cx.unwrap().exit().await.unwrap()
-}
-
-/// Sent the client used for global [request] and [notify] functions within the
-/// runtime of this future
-pub async fn with_comm<F: Future>(c: Rc<dyn Client>, ctx: CommCtx, fut: F) -> F::Output {
-	CLIENT.scope(c, CTX.scope(Rc::new(RefCell::new(Some(ctx))), fut)).await
-}
-
-task_local! {
-	pub static MUTE_REPLY: ();
-}
-
-/// Send a request through the global client's [ClientExt::request]
-pub async fn request<T: Request + UnderRoot<Root = api::ExtHostReq>>(t: T) -> T::Response {
-	let response = get_client().request(t).await.unwrap();
-	if MUTE_REPLY.try_with(|b| *b).is_err() {
-		writeln!(log("msg"), "Got response {response:?}").await;
-	}
-	response
-}
-
-/// Send a notification through the global client's [ClientExt::notify]
-pub async fn notify<T: UnderRoot<Root = api::ExtHostNotif>>(t: T) {
-	get_client().notify(t).await.unwrap()
 }
 
 pub struct SystemRecord {
-	cted: CtedObj,
+	lazy_members: Mutex<HashMap<api::TreeId, MemberRecord>>,
+	ctx: SysCtx,
 }
 
-type SystemTable = RefCell<HashMap<api::SysId, Rc<SystemRecord>>>;
+trait_set! {
-
+	pub trait WithAtomRecordCallback<'a, T> = AsyncFnOnce(
-task_local! {
+		Box<dyn AtomDynfo>,
-	static SYSTEM_TABLE: SystemTable;
+		AtomTypeId,
+		&'a [u8]
+	) -> T
 }
 
-async fn with_sys_record<F: Future>(id: api::SysId, fut: F) -> F::Output {
+pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
-	let cted = SYSTEM_TABLE.with(|tbl| tbl.borrow().get(&id).expect("Invalid sys ID").cted.clone());
+	get_sys_ctx: &impl Fn(api::SysId) -> F,
-	with_sys(SysCtx(id, cted), fut).await
+	atom: &'a api::Atom,
+	cb: impl WithAtomRecordCallback<'a, T>,
+) -> T {
+	let mut data = &atom.data.0[..];
+	let ctx = get_sys_ctx(atom.owner).await;
+	let inst = ctx.get::<CtedObj>().inst();
+	let id = AtomTypeId::decode(Pin::new(&mut data)).await;
+	let atom_record = atom_by_idx(inst.card(), id.clone()).expect("Atom ID reserved");
+	with_ctx(ctx, async move { cb(atom_record, id, data).await }).await
 }
 
-pub trait ContextModifier: 'static {
+pub struct ExtensionOwner {
-	fn apply<'a>(self: Box<Self>, fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()>;
+	_interner_cell: Rc<RefCell<Option<Interner>>>,
+	_systems_lock: Rc<RwLock<HashMap<api::SysId, SystemRecord>>>,
+	out_recv: Mutex<Receiver<Vec<u8>>>,
+	out_send: Sender<Vec<u8>>,
 }
 
-impl<F: AsyncFnOnce(LocalBoxFuture<'_, ()>) + 'static> ContextModifier for F {
+impl ExtPort for ExtensionOwner {
-	fn apply<'a>(self: Box<Self>, fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		Box::pin((self)(fut))
+		Box::pin(async { self.out_send.clone().send(msg.to_vec()).boxed_local().await.unwrap() })
+	}
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
+		Box::pin(async { self.out_recv.lock().await.next().await })
 	}
 }
 
-pub struct ExtensionBuilder {
+pub fn extension_init(
-	pub name: &'static str,
+	data: ExtensionData,
-	pub systems: Vec<Box<dyn DynSystemCtor>>,
+	host_header: api::HostHeader,
-	pub context: Vec<Box<dyn ContextModifier>>,
+	spawner: Spawner,
-}
+) -> ExtInit {
-impl ExtensionBuilder {
+	let api::HostHeader { log_strategy, msg_logs } = host_header;
-	pub fn new(name: &'static str) -> Self { Self { name, systems: Vec::new(), context: Vec::new() } }
+	let decls = (data.systems.iter().enumerate())
-	pub fn system(mut self, ctor: impl SystemCtor) -> Self {
-		self.systems.push(Box::new(ctor) as Box<_>);
-		self
-	}
-	pub fn add_context(&mut self, fun: impl ContextModifier) {
-		self.context.push(Box::new(fun) as Box<_>);
-	}
-	pub fn context(mut self, fun: impl ContextModifier) -> Self {
-		self.add_context(fun);
-		self
-	}
-	pub fn build(mut self, mut ctx: ExtPort) {
-		self.add_context(with_funs_ctx);
-		self.add_context(with_parsed_const_ctx);
-		self.add_context(with_obj_store);
-		self.add_context(with_lazy_member_store);
-		self.add_context(with_refl_roots);
-		(ctx.spawn)(Box::pin(async move {
-			let host_header = api::HostHeader::decode(ctx.input.as_mut()).await.unwrap();
-			let decls = (self.systems.iter().enumerate())
 		.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
 		.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
 		.collect_vec();
-			api::ExtensionHeader { name: self.name.to_string(), systems: decls.clone() }
+	let systems_lock = Rc::new(RwLock::new(HashMap::<api::SysId, SystemRecord>::new()));
-				.encode(ctx.output.as_mut())
+	let ext_header = api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() };
-				.await
+	let (out_send, in_recv) = channel::<Vec<u8>>(1);
-				.unwrap();
+	let (in_send, out_recv) = channel::<Vec<u8>>(1);
-			ctx.output.as_mut().flush().await.unwrap();
+	let (exit_send, exit_recv) = channel(1);
-			let logger1 = LoggerImpl::from_api(&host_header.logger);
+	let logger = Logger::new(log_strategy);
-			let logger2 = logger1.clone();
+	let msg_logger = Logger::new(msg_logs);
-			let (client, comm_ctx, extension_srv) = io_comm(ctx.output, ctx.input);
+	let interner_cell = Rc::new(RefCell::new(None::<Interner>));
-			let extension_fut = extension_srv.listen(
+	let interner_weak = Rc::downgrade(&interner_cell);
-				async |n: Box<dyn MsgReader<'_>>| {
+	let systems_weak = Rc::downgrade(&systems_lock);
-					let notif = n.read().await.unwrap();
+	let get_ctx = clone!(systems_weak; move |id: api::SysId| clone!(systems_weak; async move {
-					match notif {
+		let systems =
-						api::HostExtNotif::Exit => exit().await,
+			systems_weak.upgrade().expect("System table dropped before request processing done");
+		systems.read().await.get(&id).expect("System not found").ctx.clone()
+	}));
+	let init_ctx = {
+		clone!(interner_weak, spawner, logger);
+		move |id: api::SysId, cted: CtedObj, reqnot: ReqNot<api::ExtMsgSet>| {
+			clone!(interner_weak, spawner, logger; async move {
+				let interner_rc =
+					interner_weak.upgrade().expect("System construction order while shutting down");
+				let i = interner_rc.borrow().clone().expect("mk_ctx called very early, no interner!");
+				SysCtx::new(id, i, reqnot, spawner, logger, cted)
+			})
 		}
-					Ok(())
+	};
+	let rn = ReqNot::<api::ExtMsgSet>::new(
+		msg_logger.clone(),
+		move |a, _| {
+			clone!(in_send mut);
+			Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })
 		},
-				async |mut reader| {
+		{
-					with_stash(async {
+			clone!(exit_send);
-						let req = reader.read_req().await.unwrap();
+			move |n, _| {
-						let handle = reader.finish().await;
+				clone!(exit_send mut);
-						// Atom printing is never reported because it generates too much
+				async move {
-						// noise
+					match n {
-						if !matches!(req, api::HostExtReq::AtomReq(api::AtomReq::AtomPrint(_))) {
+						api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
-							writeln!(log("msg"), "{} extension received request {req:?}", self.name).await;
 					}
+				}
+				.boxed_local()
+			}
+		},
+		{
+			clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
+			move |hand, req| {
+				clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
+				async move {
+					let interner_cell = interner_weak.upgrade().expect("Interner dropped before request");
+					let interner =
+						interner_cell.borrow().clone().expect("Request arrived before interner set");
+					if !matches!(req, api::HostExtReq::AtomReq(api::AtomReq::AtomPrint(_))) {
+						writeln!(msg_logger, "{} extension received request {req:?}", data.name);
+					}
+
 					match req {
 						api::HostExtReq::SystemDrop(sys_drop) => {
-								SYSTEM_TABLE.with(|l| l.borrow_mut().remove(&sys_drop.0));
+							if let Some(rc) = systems_weak.upgrade() {
-								handle.reply(&sys_drop, &()).await
+								mem::drop(rc.write().await.remove(&sys_drop.0))
+							}
+							hand.handle(&sys_drop, &()).await
 						},
 						api::HostExtReq::AtomDrop(atom_drop @ api::AtomDrop(sys_id, atom)) =>
-								with_sys_record(sys_id, async {
+							with_ctx(get_ctx(sys_id).await, async move {
 								take_atom(atom).await.dyn_free().await;
-									handle.reply(&atom_drop, &()).await
+								hand.handle(&atom_drop, &()).await
 							})
 							.await,
-							api::HostExtReq::Ping(ping @ api::Ping) => handle.reply(&ping, &()).await,
+						api::HostExtReq::Ping(ping @ api::Ping) => hand.handle(&ping, &()).await,
-							api::HostExtReq::Sweep(api::Sweep) => todo!(),
+						api::HostExtReq::Sweep(sweep @ api::Sweep) =>
+							hand.handle(&sweep, &interner.sweep_replica().await).await,
 						api::HostExtReq::SysReq(api::SysReq::NewSystem(new_sys)) => {
-								let (ctor_idx, _) =
+							let (sys_id, _) = (decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system))
-									(decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system))
 								.expect("NewSystem call received for invalid system");
-								let cted = self.systems[ctor_idx].new_system(&new_sys);
+							let cted = data.systems[sys_id].new_system(&new_sys);
-								let record = Rc::new(SystemRecord { cted: cted.clone() });
+							with_ctx(init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await, async move {
-								SYSTEM_TABLE.with(|tbl| {
-									let mut g = tbl.borrow_mut();
-									g.insert(new_sys.id, record);
-								});
-								with_sys_record(new_sys.id, async {
 								let lex_filter =
 									cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
 										char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
 									});
+								let lazy_members = Mutex::new(HashMap::new());
 								let const_root = stream::iter(cted.inst().dyn_env().await)
-										.then(async |mem| {
+									.then(|mem| {
-											let name = is(&mem.name).await;
+										let lazy_mems = &lazy_members;
+										async move {
+											let name = i().i(&mem.name).await;
 											let mut tia_ctx = TreeIntoApiCtxImpl {
+												lazy_members: &mut *lazy_mems.lock().await,
 												basepath: &[],
 												path: Substack::Bottom.push(name.clone()),
 											};
 											(name.to_api(), mem.kind.into_api(&mut tia_ctx).await)
+										}
 									})
 									.collect()
 									.await;
 								let prelude =
 									cted.inst().dyn_prelude().await.iter().map(|sym| sym.to_api()).collect();
+								let record = SystemRecord { ctx: ctx(), lazy_members };
+								let systems = systems_weak.upgrade().expect("System constructed during shutdown");
+								systems.write().await.insert(new_sys.id, record);
 								let line_types = join_all(
 									(cted.inst().dyn_parsers().iter())
-											.map(async |p| is(p.line_head()).await.to_api()),
+										.map(|p| async { interner.i(p.line_head()).await.to_api() }),
 								)
 								.await;
 								let response =
 									api::NewSystemResponse { lex_filter, const_root, line_types, prelude };
-									handle.reply(&new_sys, &response).await
+								hand.handle(&new_sys, &response).await
 							})
 							.await
 						},
 						api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) =>
-								with_sys_record(sys_id, async {
+							with_ctx(get_ctx(sys_id).await, async move {
-									let (path, tree) = get_lazy(tree_id).await;
+								let systems = systems_weak.upgrade().expect("Member queried during shutdown");
-									let mut tia_ctx =
+								let systems_g = systems.read().await;
-										TreeIntoApiCtxImpl { path: Substack::Bottom, basepath: &path[..] };
+								let mut lazy_members =
-									handle.reply(&get_tree, &tree.into_api(&mut tia_ctx).await).await
+									systems_g.get(&sys_id).expect("System not found").lazy_members.lock().await;
+								let (path, cb) = match lazy_members.insert(tree_id, MemberRecord::Res) {
+									None => panic!("Tree for ID not found"),
+									Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
+									Some(MemberRecord::Gen(path, cb)) => (path, cb),
+								};
+								let tree = cb.build(Sym::new(path.clone(), &interner).await.unwrap()).await;
+								let mut tia_ctx = TreeIntoApiCtxImpl {
+									path: Substack::Bottom,
+									basepath: &path,
+									lazy_members: &mut lazy_members,
+								};
+								hand.handle(&get_tree, &tree.into_api(&mut tia_ctx).await).await
 							})
 							.await,
 						api::HostExtReq::SysReq(api::SysReq::SysFwded(fwd)) => {
-								let fwd_tok = Witness::of(&fwd);
 							let api::SysFwded(sys_id, payload) = fwd;
-								with_sys_record(sys_id, async {
+							let ctx = get_ctx(sys_id).await;
-									let mut reply = Vec::new();
+							with_ctx(ctx.clone(), async move {
-									let req = TrivialReqCycle { req: &payload, rep: &mut reply };
+								let sys = ctx.cted().inst();
-									let _ = cted().inst().dyn_request(Box::new(req)).await;
+								sys.dyn_request(hand, payload).await
-									handle.reply(fwd_tok, &reply).await
 							})
 							.await
 						},
 						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) =>
-								with_sys_record(sys, async {
+							with_ctx(get_ctx(sys).await, async move {
-									let text = es(text).await;
+								let text = Tok::from_api(text, &i()).await;
-									let src = Sym::from_api(src).await;
+								let src = Sym::from_api(src, &i()).await;
+								let rep = Reporter::new();
 								let expr_store = BorrowedExprStore::new();
-									let tail = &text[pos as usize..];
+								let trigger_char = text.chars().nth(pos as usize).unwrap();
-									let trigger_char = tail.chars().next().unwrap();
 								let ekey_na = ekey_not_applicable().await;
 								let ekey_cascade = ekey_cascade().await;
-									let lexers = cted().inst().dyn_lexers();
+								let lexers = ctx().cted().inst().dyn_lexers();
-									for lx in
+								for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char))
-										lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char))
 								{
-										let ctx = LexContext::new(&expr_store, &text, id, pos, src.clone());
+									let ctx = LexContext::new(&expr_store, &text, id, pos, src.clone(), &rep);
-										match try_with_reporter(lx.lex(tail, &ctx)).await {
+									match lx.lex(&text[pos as usize..], &ctx).await {
 										Err(e) if e.any(|e| *e == ekey_na) => continue,
 										Err(e) => {
 											let eopt = e.keep_only(|e| *e != ekey_cascade).map(|e| Err(e.to_api()));
 											expr_store.dispose().await;
-												return handle.reply(&lex, &eopt).await;
+											return hand.handle(&lex, &eopt).await;
 										},
 										Ok((s, expr)) => {
-												let expr = join_all(
+											let expr = expr.into_api(&mut (), &mut ()).await;
-													(expr.into_iter())
-														.map(|tok| async { tok.into_api(&mut (), &mut ()).await }),
-												)
-												.await;
 											let pos = (text.len() - s.len()) as u32;
 											expr_store.dispose().await;
-												return handle.reply(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
+											return hand.handle(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
 										},
 									}
 								}
-									writeln!(log("warn"), "Got notified about n/a character '{trigger_char}'").await;
+								writeln!(logger, "Got notified about n/a character '{trigger_char}'");
 								expr_store.dispose().await;
-									handle.reply(&lex, &None).await
+								hand.handle(&lex, &None).await
 							})
 							.await,
 						api::HostExtReq::ParseLine(pline) => {
 							let api::ParseLine { module, src, exported, comments, sys, line, idx } = &pline;
-								with_sys_record(*sys, async {
+							with_ctx(get_ctx(*sys).await, async {
-									let parsers = cted().inst().dyn_parsers();
+								let parsers = ctx().cted().inst().dyn_parsers();
-									let src = Sym::from_api(*src).await;
+								let src = Sym::from_api(*src, &i()).await;
 								let comments =
-										join_all(comments.iter().map(|c| Comment::from_api(c, src.clone()))).await;
+									join_all(comments.iter().map(|c| Comment::from_api(c, src.clone(), &interner)))
+										.await;
 								let expr_store = BorrowedExprStore::new();
 								let line: Vec<PTokTree> =
-										ttv_from_api(line, &mut &expr_store, &mut (), &src).await;
+									ttv_from_api(line, &mut &expr_store, &mut (), &src, &i()).await;
 								let snip = Snippet::new(line.first().expect("Empty line"), &line);
 								let parser = parsers[*idx as usize];
-									let module = Sym::from_api(*module).await;
+								let module = Sym::from_api(*module, &i()).await;
-									let pctx = ParsCtx::new(module);
+								let reporter = Reporter::new();
-									let o_line =
+								let pctx = ParsCtx::new(module, &reporter);
-										match try_with_reporter(parser.parse(pctx, *exported, comments, snip)).await {
+								let parse_res = parser.parse(pctx, *exported, comments, snip).await;
+								let o_line = match reporter.merge(parse_res) {
 									Err(e) => Err(e.to_api()),
 									Ok(t) => Ok(linev_into_api(t).await),
 								};
 								mem::drop(line);
 								expr_store.dispose().await;
-									handle.reply(&pline, &o_line).await
+								hand.handle(&pline, &o_line).await
 							})
 							.await
 						},
 						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst(sys, id)) =>
-								with_sys_record(sys, async {
+							with_ctx(get_ctx(sys).await, async move {
 								let cnst = get_const(id).await;
-									handle.reply(fpc, &cnst.serialize().await).await
+								hand.handle(fpc, &cnst.serialize().await).await
 							})
 							.await,
 						api::HostExtReq::AtomReq(atom_req) => {
 							let atom = atom_req.get_atom();
-								with_sys_record(atom.owner, async {
+							let atom_req = atom_req.clone();
-									let (nfo, id, buf) = resolve_atom_type(atom);
+							with_atom_record(&get_ctx, atom, async move |nfo, id, buf| {
 								let actx = AtomCtx(buf, atom.drop);
 								match &atom_req {
 									api::AtomReq::SerializeAtom(ser) => {
-											let mut buf = enc_vec(&id);
+										let mut buf = enc_vec(&id).await;
 										match nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await {
-												None => handle.reply(ser, &None).await,
+											None => hand.handle(ser, &None).await,
 											Some(refs) => {
 												let refs =
-														join_all(refs.into_iter().map(async |ex| ex.into_api(&mut ()).await))
+													join_all(refs.into_iter().map(|ex| async { ex.into_api(&mut ()).await }))
 														.await;
-													handle.reply(ser, &Some((buf, refs))).await
+												hand.handle(ser, &Some((buf, refs))).await
 											},
 										}
 									},
 									api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
-											handle.reply(print, &nfo.print(actx).await.to_api()).await,
+										hand.handle(print, &nfo.print(actx).await.to_api()).await,
 									api::AtomReq::Fwded(fwded) => {
 										let api::Fwded(_, key, payload) = &fwded;
 										let mut reply = Vec::new();
-											let key = Sym::from_api(*key).await;
+										let key = Sym::from_api(*key, &interner).await;
-											let req = TrivialReqCycle { req: payload, rep: &mut reply };
+										let some = nfo
-											let some = nfo.handle_req(actx, key, Box::new(req)).await;
+											.handle_req(
-											handle.reply(fwded, &some.then_some(reply)).await
+												actx,
+												key,
+												Pin::<&mut &[u8]>::new(&mut &payload[..]),
+												Pin::<&mut Vec<_>>::new(&mut reply),
+											)
+											.await;
+										hand.handle(fwded, &some.then_some(reply)).await
 									},
 									api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
 										let expr_store = BorrowedExprStore::new();
@@ -330,7 +356,7 @@ impl ExtensionBuilder {
 										let api_expr = ret.serialize().await;
 										mem::drop(expr_handle);
 										expr_store.dispose().await;
-											handle.reply(call, &api_expr).await
+										hand.handle(call, &api_expr).await
 									},
 									api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
 										let expr_store = BorrowedExprStore::new();
@@ -339,15 +365,15 @@ impl ExtensionBuilder {
 										let api_expr = ret.serialize().await;
 										mem::drop(expr_handle);
 										expr_store.dispose().await;
-											handle.reply(call, &api_expr).await
+										hand.handle(call, &api_expr).await
 									},
 									api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
-											Err(e) => handle.reply(cmd, &Err(e.to_api())).await,
+										Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
 										Ok(opt) => match opt {
-												None => handle.reply(cmd, &Ok(api::NextStep::Halt)).await,
+											None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
 											Some(cont) => {
 												let cont = cont.serialize().await;
-													handle.reply(cmd, &Ok(api::NextStep::Continue(cont))).await
+												hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
 											},
 										},
 									},
@@ -357,45 +383,41 @@ impl ExtensionBuilder {
 						},
 						api::HostExtReq::DeserAtom(deser) => {
 							let api::DeserAtom(sys, buf, refs) = &deser;
-								let read = &mut &buf[..];
+							let mut read = &mut &buf[..];
-								with_sys_record(*sys, async {
+							let ctx = get_ctx(*sys).await;
 							// SAFETY: deserialization implicitly grants ownership to previously owned exprs
 							let refs = (refs.iter())
 								.map(|tk| Expr::from_handle(ExprHandle::deserialize(*tk)))
 								.collect_vec();
-									let id = AtomTypeId::decode_slice(read);
+							let id = AtomTypeId::decode(Pin::new(&mut read)).await;
-									let nfo = atom_by_idx(cted().inst().card(), id)
+							let inst = ctx.cted().inst();
-										.expect("Deserializing atom with invalid ID");
+							let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
-									handle.reply(&deser, &nfo.deserialize(read, &refs).await).await
+							hand.handle(&deser, &nfo.deserialize(read, &refs).await).await
-								})
-								.await
 						},
 					}
-					})
+				}
-					.await
+				.boxed_local()
+			}
 		},
 	);
-			// add essential services to the very tail, then fold all context into the run
+	*interner_cell.borrow_mut() =
-			// future
+		Some(Interner::new_replica(rn.clone().map(|ir: api::IntReq| ir.into_root())));
-			SYSTEM_TABLE
+	spawner(Box::pin(clone!(spawner; async move {
-				.scope(
+		let mut streams = stream_select! { in_recv.map(Some), exit_recv.map(|_| None) };
-					RefCell::default(),
+		while let Some(item) = streams.next().await {
-					with_interner(
+			match item {
-						new_interner(),
+				Some(rcvd) => spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd[..]).await }))),
-						with_logger(
+				None => break,
-							logger2,
+			}
-							with_comm(
+		}
-								Rc::new(client),
+	})));
-								comm_ctx,
+	ExtInit {
-								(self.context.into_iter()).fold(
+		header: ext_header,
-									Box::pin(async { extension_fut.await.unwrap() }) as LocalBoxFuture<()>,
+		port: Box::new(ExtensionOwner {
-									|fut, cx| cx.apply(fut),
+			out_recv: Mutex::new(out_recv),
-								),
+			out_send,
-							),
+			_interner_cell: interner_cell,
-						),
+			_systems_lock: systems_lock,
-					),
+		}),
-				)
-				.await;
-		}) as Pin<Box<_>>);
 	}
 }

orchid-extension/src/expr.rs

@@ -2,7 +2,6 @@ use std::cell::RefCell;
 use std::fmt;
 use std::hash::Hash;
 use std::rc::Rc;
-use std::thread::panicking;
 
 use async_once_cell::OnceCell;
 use derive_destructure::destructure;
@@ -10,13 +9,12 @@ use hashbrown::HashSet;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::location::Pos;
-use orchid_base::stash::stash;
+use orchid_base::reqnot::Requester;
 
 use crate::api;
 use crate::atom::ForeignAtom;
-use crate::entrypoint::{notify, request};
+use crate::context::{ctx, i};
 use crate::gen_expr::{GExpr, GExprKind};
-use crate::system::sys_id;
 
 pub struct BorrowedExprStore(RefCell<Option<HashSet<Rc<ExprHandle>>>>);
 impl BorrowedExprStore {
@@ -30,7 +28,7 @@ impl BorrowedExprStore {
 }
 impl Drop for BorrowedExprStore {
 	fn drop(&mut self) {
-		if self.0.borrow().is_some() && !panicking() {
+		if self.0.borrow().is_some() {
 			panic!("This should always be explicitly disposed")
 		}
 	}
@@ -74,7 +72,7 @@ impl ExprHandle {
 	/// to lend the expr, and you expect the receiver to use
 	/// [ExprHandle::borrowed] or [ExprHandle::from_ticket]
 	pub fn ticket(&self) -> api::ExprTicket { self.0 }
-	async fn send_acq(&self) { notify(api::Acquire(sys_id(), self.0)).await }
+	async fn send_acq(&self) { ctx().reqnot().notify(api::Acquire(ctx().sys_id(), self.0)).await }
 	/// If this is the last one reference, do nothing, otherwise send an Acquire
 	pub async fn on_borrow_expire(self: Rc<Self>) { self.serialize().await; }
 	/// Drop the handle and get the ticket without a release notification.
@@ -95,8 +93,8 @@ impl fmt::Debug for ExprHandle {
 }
 impl Drop for ExprHandle {
 	fn drop(&mut self) {
-		let notif = api::Release(sys_id(), self.0);
+		let notif = api::Release(ctx().sys_id(), self.0);
-		stash(async move { notify(notif).await })
+		ctx().spawn(async move { ctx().reqnot().clone().notify(notif).await })
 	}
 }
 
@@ -118,12 +116,12 @@ impl Expr {
 	}
 	pub async fn data(&self) -> &ExprData {
 		(self.data.get_or_init(async {
-			let details = request(api::Inspect { target: self.handle.ticket() }).await;
+			let details = ctx().reqnot().request(api::Inspect { target: self.handle.ticket() }).await;
-			let pos = Pos::from_api(&details.location).await;
+			let pos = Pos::from_api(&details.location, &i()).await;
 			let kind = match details.kind {
 				api::InspectedKind::Atom(a) =>
 					ExprKind::Atom(ForeignAtom::new(self.handle.clone(), a, pos.clone())),
-				api::InspectedKind::Bottom(b) => ExprKind::Bottom(OrcErrv::from_api(&b).await),
+				api::InspectedKind::Bottom(b) => ExprKind::Bottom(OrcErrv::from_api(&b, &i()).await),
 				api::InspectedKind::Opaque => ExprKind::Opaque,
 			};
 			ExprData { pos, kind }
@@ -136,7 +134,6 @@ impl Expr {
 			_ => Err(self),
 		}
 	}
-	pub async fn pos(&self) -> Pos { self.data().await.pos.clone() }
 	pub fn handle(&self) -> Rc<ExprHandle> { self.handle.clone() }
 
 	pub fn slot(&self) -> GExpr {
@@ -152,7 +149,8 @@ impl Format for Expr {
 		match &self.data().await.kind {
 			ExprKind::Opaque => "OPAQUE".to_string().into(),
 			ExprKind::Bottom(b) => format!("Bottom({b})").into(),
-			ExprKind::Atom(a) => FmtUnit::from_api(&request(api::ExtAtomPrint(a.atom.clone())).await),
+			ExprKind::Atom(a) =>
+				FmtUnit::from_api(&ctx().reqnot().request(api::ExtAtomPrint(a.atom.clone())).await),
 		}
 	}
 }

orchid-extension/src/ext_port.rs

@@ -1,12 +0,0 @@
-use std::pin::Pin;
-use std::rc::Rc;
-
-use futures::future::LocalBoxFuture;
-use futures::{AsyncRead, AsyncWrite};
-
-pub struct ExtPort {
-	pub input: Pin<Box<dyn AsyncRead>>,
-	pub output: Pin<Box<dyn AsyncWrite>>,
-	pub log: Pin<Box<dyn AsyncWrite>>,
-	pub spawn: Rc<dyn Fn(LocalBoxFuture<'static, ()>)>,
-}

orchid-extension/src/func_atom.rs

@@ -1,12 +1,12 @@
 use std::any::TypeId;
 use std::borrow::Cow;
-use std::cell::RefCell;
 use std::collections::HashMap;
 use std::future::Future;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use futures::future::{LocalBoxFuture, join_all};
+use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
 use futures::{AsyncWrite, FutureExt};
 use itertools::Itertools;
 use never::Never;
@@ -14,60 +14,29 @@ use orchid_api_traits::Encode;
 use orchid_base::clone;
 use orchid_base::error::OrcRes;
 use orchid_base::format::{FmtCtx, FmtUnit};
-use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use task_local::task_local;
 use trait_set::trait_set;
 
-use crate::api;
 use crate::atom::Atomic;
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
+use crate::context::{SysCtxEntry, ctx, i};
 use crate::conv::ToExpr;
 use crate::coroutine_exec::{ExecHandle, exec};
 use crate::expr::Expr;
-use crate::gen_expr::{GExpr, new_atom};
+use crate::gen_expr::GExpr;
-use crate::system::sys_id;
 
 trait_set! {
 	trait FunCB = Fn(Vec<Expr>) -> LocalBoxFuture<'static, OrcRes<GExpr>> + 'static;
 }
 
-task_local! {
-	static ARGV: Vec<Expr>;
-}
-
-pub fn get_arg(idx: usize) -> Expr {
-	ARGV
-		.try_with(|argv| {
-			(argv.get(idx).cloned())
-				.unwrap_or_else(|| panic!("Cannot read argument ##{idx}, only have {}", argv.len()))
-		})
-		.expect("get_arg called outside ExprFunc")
-}
-
-pub fn get_argc() -> usize {
-	ARGV.try_with(|argv| argv.len()).expect("get_arg called outside ExprFunc")
-}
-
-pub async fn get_arg_posv(idxes: impl IntoIterator<Item = usize>) -> impl Iterator<Item = Pos> {
-	let args = (ARGV.try_with(|argv| idxes.into_iter().map(|i| &argv[i]).cloned().collect_vec()))
-		.expect("get_arg_posv called outside ExprFunc");
-	join_all(args.iter().map(|expr| expr.pos())).await.into_iter()
-}
-
 pub trait ExprFunc<I, O>: Clone + 'static {
 	fn argtyps() -> &'static [TypeId];
 	fn apply<'a>(&self, hand: ExecHandle<'a>, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
 }
 
-task_local! {
+#[derive(Default)]
-	static FUNS_CTX: RefCell<HashMap<(api::SysId, Sym), FunRecord>>;
+struct FunsCtx(Mutex<HashMap<Sym, FunRecord>>);
-}
+impl SysCtxEntry for FunsCtx {}
-
-pub(crate) fn with_funs_ctx<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
-	Box::pin(FUNS_CTX.scope(RefCell::default(), fut))
-}
-
 #[derive(Clone)]
 struct FunRecord {
 	argtyps: &'static [TypeId],
@@ -81,9 +50,7 @@ fn process_args<I, O, F: ExprFunc<I, O>>(f: F) -> FunRecord {
 		exec(async move |mut hand| {
 			let mut norm_args = Vec::with_capacity(v.len());
 			for (expr, typ) in v.into_iter().zip(argtyps) {
-				if *typ == TypeId::of::<Expr>() {
+				if *typ != TypeId::of::<Expr>() {
-					norm_args.push(expr);
-				} else {
 					norm_args.push(hand.exec(expr).await?);
 				}
 			}
@@ -108,17 +75,17 @@ pub(crate) struct Fun {
 }
 impl Fun {
 	pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, f: F) -> Self {
-		FUNS_CTX.with(|cx| {
+		let ctx = ctx();
-			let mut fung = cx.borrow_mut();
+		let funs: &FunsCtx = ctx.get_or_default();
-			let record = if let Some(record) = fung.get(&(sys_id(), path.clone())) {
+		let mut fung = funs.0.lock().await;
+		let record = if let Some(record) = fung.get(&path) {
 			record.clone()
 		} else {
 			let record = process_args(f);
-				fung.insert((sys_id(), path.clone()), record.clone());
+			fung.insert(path.clone(), record.clone());
 			record
 		};
 		Self { args: vec![], path, record }
-		})
 	}
 	pub fn arity(&self) -> u8 { self.record.argtyps.len() as u8 }
 }
@@ -134,17 +101,17 @@ impl OwnedAtom for Fun {
 		if new_args.len() == self.record.argtyps.len() {
 			(self.record.fun)(new_args).await.to_gen().await
 		} else {
-			new_atom(Self { args: new_args, record: self.record.clone(), path: self.path.clone() })
+			Self { args: new_args, record: self.record.clone(), path: self.path.clone() }.to_gen().await
 		}
 	}
 	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
 	async fn serialize(&self, write: Pin<&mut (impl AsyncWrite + ?Sized)>) -> Self::Refs {
-		self.path.to_api().encode(write).await.unwrap();
+		self.path.to_api().encode(write).await;
 		self.args.clone()
 	}
 	async fn deserialize(mut ds_cx: impl DeserializeCtx, args: Self::Refs) -> Self {
-		let path = Sym::from_api(ds_cx.decode().await).await;
+		let path = Sym::from_api(ds_cx.decode().await, &i()).await;
-		let record = (FUNS_CTX.with(|funs| funs.borrow().get(&(sys_id(), path.clone())).cloned()))
+		let record = (ctx().get::<FunsCtx>().0.lock().await.get(&path))
 			.expect("Function missing during deserialization")
 			.clone();
 		Self { args, path, record }
@@ -180,7 +147,7 @@ impl OwnedAtom for Lambda {
 		if new_args.len() == self.record.argtyps.len() {
 			(self.record.fun)(new_args).await.to_gen().await
 		} else {
-			new_atom(Self { args: new_args, record: self.record.clone() })
+			Self { args: new_args, record: self.record.clone() }.to_gen().await
 		}
 	}
 	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
@@ -192,7 +159,7 @@ mod expr_func_derives {
 
 	use orchid_base::error::OrcRes;
 
-	use super::{ARGV, ExprFunc};
+	use super::ExprFunc;
 	use crate::conv::{ToExpr, TryFromExpr};
 	use crate::func_atom::{ExecHandle, Expr};
 	use crate::gen_expr::GExpr;
@@ -211,9 +178,8 @@ mod expr_func_derives {
           }
           async fn apply<'a>(&self, _: ExecHandle<'a>, v: Vec<Expr>) -> OrcRes<GExpr> {
             assert_eq!(v.len(), Self::argtyps().len(), "Arity mismatch");
-						let argv = v.clone();
             let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above"));
-            Ok(ARGV.scope(argv, self($($t::try_from_expr([< $t:lower >]).await?,)*)).await.to_gen().await)
+            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_gen().await)
           }
         }
       }

orchid-extension/src/gen_expr.rs

@@ -6,11 +6,12 @@ use orchid_base::error::{OrcErr, OrcErrv};
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
+use orchid_base::reqnot::Requester;
 use orchid_base::{match_mapping, tl_cache};
 
 use crate::api;
 use crate::atom::{AtomFactory, ToAtom};
-use crate::entrypoint::request;
+use crate::context::ctx;
 use crate::expr::Expr;
 
 #[derive(Clone, Debug)]
@@ -39,7 +40,7 @@ impl GExpr {
 	}
 	pub fn at(self, pos: Pos) -> Self { GExpr { pos, kind: self.kind } }
 	pub async fn create(self) -> Expr {
-		Expr::deserialize(request(api::Create(self.serialize().await)).await).await
+		Expr::deserialize(ctx().reqnot().request(api::Create(self.serialize().await)).await).await
 	}
 }
 impl Format for GExpr {
@@ -106,8 +107,7 @@ impl Format for GExprKind {
 fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
 
 pub fn sym_ref(path: Sym) -> GExpr { inherit(GExprKind::Const(path)) }
-/// Creates an expression from a new atom that we own.
+pub fn atom<A: ToAtom>(atom: A) -> GExpr { inherit(GExprKind::NewAtom(atom.to_atom_factory())) }
-pub fn new_atom<A: ToAtom>(atom: A) -> GExpr { inherit(GExprKind::NewAtom(atom.to_atom_factory())) }
 
 pub fn seq(deps: impl IntoIterator<Item = GExpr>, val: GExpr) -> GExpr {
 	fn recur(mut ops: impl Iterator<Item = GExpr>) -> Option<GExpr> {

orchid-extension/src/interner.rs

@@ -1,50 +0,0 @@
-use std::rc::Rc;
-
-use futures::future::{LocalBoxFuture, join_all, ready};
-use itertools::Itertools;
-use orchid_base::interner::local_interner::{Int, StrBranch, StrvBranch};
-use orchid_base::interner::{IStr, IStrv, InternerSrv};
-
-use crate::api;
-use crate::entrypoint::{MUTE_REPLY, request};
-
-#[derive(Default)]
-struct ExtInterner {
-	str: Int<StrBranch>,
-	strv: Int<StrvBranch>,
-}
-impl InternerSrv for ExtInterner {
-	fn is<'a>(&'a self, v: &'a str) -> LocalBoxFuture<'a, IStr> {
-		match self.str.i(v) {
-			Ok(i) => Box::pin(ready(i)),
-			Err(e) => Box::pin(async {
-				e.set_if_empty(MUTE_REPLY.scope((), request(api::InternStr(v.to_owned()))).await)
-			}),
-		}
-	}
-	fn es(&self, t: api::TStr) -> LocalBoxFuture<'_, IStr> {
-		match self.str.e(t) {
-			Ok(i) => Box::pin(ready(i)),
-			Err(e) => Box::pin(async move { e.set_if_empty(Rc::new(request(api::ExternStr(t)).await)) }),
-		}
-	}
-	fn iv<'a>(&'a self, v: &'a [IStr]) -> LocalBoxFuture<'a, IStrv> {
-		match self.strv.i(v) {
-			Ok(i) => Box::pin(ready(i)),
-			Err(e) => Box::pin(async {
-				e.set_if_empty(request(api::InternStrv(v.iter().map(|is| is.to_api()).collect_vec())).await)
-			}),
-		}
-	}
-	fn ev(&self, t: orchid_api::TStrv) -> LocalBoxFuture<'_, IStrv> {
-		match self.strv.e(t) {
-			Ok(i) => Box::pin(ready(i)),
-			Err(e) => Box::pin(async move {
-				let tstr_v = request(api::ExternStrv(t)).await;
-				e.set_if_empty(Rc::new(join_all(tstr_v.into_iter().map(|t| self.es(t))).await))
-			}),
-		}
-	}
-}
-
-pub fn new_interner() -> Rc<dyn InternerSrv> { Rc::<ExtInterner>::default() }

orchid-extension/src/lexer.rs

@@ -1,24 +1,27 @@
 use std::fmt;
-use std::fmt::Debug;
 use std::future::Future;
 use std::ops::RangeInclusive;
 
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_errv};
-use orchid_base::interner::{IStr, is};
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::{Pos, SrcRange};
 use orchid_base::name::Sym;
+use orchid_base::parse::ParseCtx;
+use orchid_base::reqnot::Requester;
 
 use crate::api;
-use crate::entrypoint::request;
+use crate::context::{ctx, i};
 use crate::expr::BorrowedExprStore;
 use crate::parser::PTokTree;
 use crate::tree::GenTokTree;
 
-pub async fn ekey_cascade() -> IStr { is("An error cascading from a recursive call").await }
+pub async fn ekey_cascade() -> Tok<String> {
-pub async fn ekey_not_applicable() -> IStr {
+	i().i("An error cascading from a recursive call").await
-	is("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
+}
+pub async fn ekey_not_applicable() -> Tok<String> {
+	i().i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
 }
 const MSG_INTERNAL_ERROR: &str = "This error is a sentinel for the extension library.\
 	it should not be emitted by the extension.";
@@ -33,20 +36,23 @@ pub async fn err_not_applicable() -> OrcErrv {
 
 pub struct LexContext<'a> {
 	pub(crate) exprs: &'a BorrowedExprStore,
-	pub text: &'a IStr,
+	pub text: &'a Tok<String>,
 	pub id: api::ParsId,
 	pub pos: u32,
+	i: Interner,
 	pub(crate) src: Sym,
+	pub(crate) rep: &'a Reporter,
 }
 impl<'a> LexContext<'a> {
 	pub fn new(
 		exprs: &'a BorrowedExprStore,
-		text: &'a IStr,
+		text: &'a Tok<String>,
 		id: api::ParsId,
 		pos: u32,
 		src: Sym,
+		rep: &'a Reporter,
 	) -> Self {
-		Self { exprs, id, pos, src, text }
+		Self { exprs, i: i(), id, pos, rep, src, text }
 	}
 	pub fn src(&self) -> &Sym { &self.src }
 	/// This function returns [PTokTree] because it can never return
@@ -55,10 +61,10 @@ impl<'a> LexContext<'a> {
 	/// for embedding in the return value.
 	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, PTokTree)> {
 		let start = self.pos(tail);
-		let Some(lx) = request(api::SubLex { pos: start, id: self.id }).await else {
+		let Some(lx) = ctx().reqnot().request(api::SubLex { pos: start, id: self.id }).await else {
 			return Err(err_cascade().await);
 		};
-		let tree = PTokTree::from_api(&lx.tree, &mut { self.exprs }, &mut (), &self.src).await;
+		let tree = PTokTree::from_api(&lx.tree, &mut { self.exprs }, &mut (), &self.src, &i()).await;
 		Ok((&self.text[lx.pos as usize..], tree))
 	}
 
@@ -71,35 +77,26 @@ impl<'a> LexContext<'a> {
 		SrcRange::new(self.pos(tail) - len.try_into().unwrap()..self.pos(tail), &self.src)
 	}
 }
-
+impl ParseCtx for LexContext<'_> {
-pub trait LexedData {
+	fn i(&self) -> &Interner { &self.i }
-	fn into_vec(self) -> Vec<GenTokTree>;
+	fn rep(&self) -> &Reporter { self.rep }
-}
-impl LexedData for GenTokTree {
-	fn into_vec(self) -> Vec<GenTokTree> { vec![self] }
-}
-impl LexedData for Vec<GenTokTree> {
-	fn into_vec(self) -> Vec<GenTokTree> { self }
-}
-impl<const N: usize> LexedData for [GenTokTree; N] {
-	fn into_vec(self) -> Vec<GenTokTree> { self.to_vec() }
 }
 
-pub trait Lexer: Debug + Send + Sync + Sized + Default + 'static {
+pub trait Lexer: Send + Sync + Sized + Default + 'static {
 	const CHAR_FILTER: &'static [RangeInclusive<char>];
 	fn lex<'a>(
 		tail: &'a str,
 		lctx: &'a LexContext<'a>,
-	) -> impl Future<Output = OrcRes<(&'a str, impl LexedData)>>;
+	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
 }
 
-pub trait DynLexer: Debug + Send + Sync + 'static {
+pub trait DynLexer: Send + Sync + 'static {
 	fn char_filter(&self) -> &'static [RangeInclusive<char>];
 	fn lex<'a>(
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>>;
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -108,8 +105,8 @@ impl<T: Lexer> DynLexer for T {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, Vec<GenTokTree>)>> {
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>> {
-		async { T::lex(tail, ctx).await.map(|(s, d)| (s, d.into_vec())) }.boxed_local()
+		T::lex(tail, ctx).boxed_local()
 	}
 }
 

orchid-extension/src/lib.rs

@@ -3,23 +3,19 @@ use orchid_api as api;
 pub mod atom;
 pub mod atom_owned;
 pub mod atom_thin;
-pub mod binary;
 pub mod conv;
 pub mod coroutine_exec;
 pub mod entrypoint;
 pub mod expr;
-pub mod ext_port;
 pub mod func_atom;
 pub mod gen_expr;
-pub mod interner;
 pub mod lexer;
-pub mod logger;
+// pub mod msg;
+pub mod context;
 pub mod other_system;
 pub mod parser;
 pub mod reflection;
-pub mod stream_reqs;
 pub mod system;
 pub mod system_ctor;
 pub mod tokio;
 pub mod tree;
-mod trivial_req;

orchid-extension/src/logger.rs

@@ -1,57 +0,0 @@
-use std::fmt::Arguments;
-use std::fs::File;
-use std::io::Write;
-use std::rc::Rc;
-
-use futures::future::LocalBoxFuture;
-use hashbrown::HashMap;
-use orchid_base::interner::is;
-use orchid_base::logging::{LogWriter, Logger};
-
-use crate::api;
-use crate::entrypoint::notify;
-
-pub struct LogWriterImpl {
-	category: String,
-	strat: api::LogStrategy,
-}
-impl LogWriter for LogWriterImpl {
-	fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()> {
-		Box::pin(async move {
-			match &self.strat {
-				api::LogStrategy::Discard => (),
-				api::LogStrategy::Default =>
-					notify(api::Log { category: is(&self.category).await.to_api(), message: fmt.to_string() })
-						.await,
-				api::LogStrategy::File { path, .. } => {
-					let mut file = (File::options().write(true).create(true).truncate(false).open(path))
-						.unwrap_or_else(|e| panic!("Could not open {path}: {e}"));
-					file.write_fmt(fmt).unwrap_or_else(|e| panic!("Could not write to {path}: {e}"));
-				},
-			}
-		})
-	}
-}
-
-#[derive(Clone)]
-pub struct LoggerImpl {
-	default: Option<api::LogStrategy>,
-	routing: HashMap<String, api::LogStrategy>,
-}
-impl LoggerImpl {
-	pub fn from_api(api: &api::Logger) -> Self {
-		Self {
-			default: api.default.clone(),
-			routing: api.routing.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
-		}
-	}
-}
-impl Logger for LoggerImpl {
-	fn writer(&self, category: &str) -> Rc<dyn LogWriter> {
-		Rc::new(LogWriterImpl { category: category.to_string(), strat: self.strat(category) })
-	}
-	fn strat(&self, category: &str) -> orchid_api::LogStrategy {
-		(self.routing.get(category).cloned().or(self.default.clone()))
-			.expect("Unrecognized log category with no default strategy")
-	}
-}

orchid-extension/src/other_system.rs

@@ -1,7 +1,6 @@
 use crate::api;
 use crate::system::{DynSystemCard, SystemCard};
 
-#[derive(Debug)]
 pub struct SystemHandle<C: SystemCard> {
 	pub(crate) card: C,
 	pub(crate) id: api::SysId,

orchid-extension/src/parser.rs

@@ -5,22 +5,21 @@ use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, Stream, StreamExt};
 use itertools::Itertools;
 use never::Never;
-use orchid_base::error::{OrcErrv, OrcRes};
+use orchid_base::error::{OrcErrv, OrcRes, Reporter};
 use orchid_base::id_store::IdStore;
-use orchid_base::interner::IStr;
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::SrcRange;
 use orchid_base::match_mapping;
 use orchid_base::name::Sym;
-use orchid_base::parse::{Comment, Snippet};
+use orchid_base::parse::{Comment, ParseCtx, Snippet};
+use orchid_base::reqnot::Requester;
 use orchid_base::tree::{TokTree, Token, ttv_into_api};
-use task_local::task_local;
 
 use crate::api;
+use crate::context::{SysCtxEntry, ctx, i};
 use crate::conv::ToExpr;
-use crate::entrypoint::request;
 use crate::expr::Expr;
 use crate::gen_expr::GExpr;
-use crate::system::sys_id;
 use crate::tree::{GenTok, GenTokTree};
 
 pub type PTok = Token<Expr, Never>;
@@ -83,21 +82,27 @@ pub type ParserObj = &'static dyn DynParser;
 pub struct ParsCtx<'a> {
 	_parse: PhantomData<&'a mut ()>,
 	module: Sym,
+	reporter: &'a Reporter,
+	i: Interner,
 }
 impl<'a> ParsCtx<'a> {
-	pub(crate) fn new(module: Sym) -> Self { Self { _parse: PhantomData, module } }
+	pub(crate) fn new(module: Sym, reporter: &'a Reporter) -> Self {
+		Self { _parse: PhantomData, module, reporter, i: i() }
+	}
 	pub fn module(&self) -> Sym { self.module.clone() }
 }
+impl ParseCtx for ParsCtx<'_> {
+	fn i(&self) -> &Interner { &self.i }
+	fn rep(&self) -> &Reporter { self.reporter }
+}
 
 type BoxConstCallback = Box<dyn FnOnce(ConstCtx) -> LocalBoxFuture<'static, GExpr>>;
 
-task_local! {
+#[derive(Default)]
-	static PARSED_CONST_CTX: IdStore<BoxConstCallback>
+pub(crate) struct ParsedConstCtxEntry {
-}
+	pub(crate) consts: IdStore<BoxConstCallback>,
-
-pub(crate) fn with_parsed_const_ctx<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
-	Box::pin(PARSED_CONST_CTX.scope(IdStore::default(), fut))
 }
+impl SysCtxEntry for ParsedConstCtxEntry {}
 
 pub struct ParsedLine {
 	pub sr: SrcRange,
@@ -109,7 +114,7 @@ impl ParsedLine {
 		sr: &SrcRange,
 		comments: impl IntoIterator<Item = &'a Comment>,
 		exported: bool,
-		name: IStr,
+		name: Tok<String>,
 		f: F,
 	) -> Self {
 		let cb = Box::new(|ctx| async move { f(ctx).await.to_gen().await }.boxed_local());
@@ -121,7 +126,7 @@ impl ParsedLine {
 		sr: &SrcRange,
 		comments: impl IntoIterator<Item = &'a Comment>,
 		exported: bool,
-		name: &IStr,
+		name: &Tok<String>,
 		use_prelude: bool,
 		lines: impl IntoIterator<Item = ParsedLine>,
 	) -> Self {
@@ -140,7 +145,7 @@ impl ParsedLine {
 					exported: mem.exported,
 					kind: match mem.kind {
 						ParsedMemKind::Const(cb) => api::ParsedMemberKind::Constant(api::ParsedConstId(
-							PARSED_CONST_CTX.with(|consts| consts.add(cb).id()),
+							ctx().get_or_default::<ParsedConstCtxEntry>().consts.add(cb).id(),
 						)),
 						ParsedMemKind::Mod { lines, use_prelude } => api::ParsedMemberKind::Module {
 							lines: linev_into_api(lines).boxed_local().await,
@@ -165,7 +170,7 @@ pub enum ParsedLineKind {
 }
 
 pub struct ParsedMem {
-	pub name: IStr,
+	pub name: Tok<String>,
 	pub exported: bool,
 	pub kind: ParsedMemKind,
 }
@@ -184,16 +189,16 @@ impl ConstCtx {
 		&'b self,
 		names: impl IntoIterator<Item = &'b Sym> + 'b,
 	) -> impl Stream<Item = OrcRes<Sym>> + 'b {
-		let names = names.into_iter().map(|n| n.to_api()).collect_vec();
+		let resolve_names = api::ResolveNames {
+			constid: self.constid,
+			sys: ctx().sys_id(),
+			names: names.into_iter().map(|n| n.to_api()).collect_vec(),
+		};
 		stream(async |mut cx| {
-			if names.is_empty() {
+			for name_opt in ctx().reqnot().request(resolve_names).await {
-				return;
-			}
-			let resolve_names = api::ResolveNames { constid: self.constid, sys: sys_id(), names };
-			for name_opt in request(resolve_names).await {
 				cx.emit(match name_opt {
-					Err(e) => Err(OrcErrv::from_api(&e).await),
+					Err(e) => Err(OrcErrv::from_api(&e, &i()).await),
-					Ok(name) => Ok(Sym::from_api(name).await),
+					Ok(name) => Ok(Sym::from_api(name, &i()).await),
 				})
 				.await
 			}
@@ -205,7 +210,8 @@ impl ConstCtx {
 }
 
 pub(crate) async fn get_const(id: api::ParsedConstId) -> GExpr {
-	let cb = PARSED_CONST_CTX
+	let cb = (ctx().get_or_default::<ParsedConstCtxEntry>().consts.get(id.0))
-		.with(|ent| ent.get(id.0).expect("Bad ID or double read of parsed const").remove());
+		.expect("Bad ID or double read of parsed const")
+		.remove();
 	cb(ConstCtx { constid: id }).await
 }

orchid-extension/src/reflection.rs

@@ -1,18 +1,15 @@
-use std::cell::{OnceCell, RefCell};
+use std::cell::OnceCell;
 use std::rc::Rc;
 
 use futures::FutureExt;
-use futures::future::LocalBoxFuture;
 use futures::lock::Mutex;
-use hashbrown::HashMap;
 use memo_map::MemoMap;
-use orchid_base::interner::{IStr, es, iv};
+use orchid_base::interner::Tok;
 use orchid_base::name::{NameLike, VPath};
-use task_local::task_local;
+use orchid_base::reqnot::Requester;
 
 use crate::api;
-use crate::entrypoint::request;
+use crate::context::{SysCtxEntry, ctx, i};
-use crate::system::sys_id;
 
 #[derive(Debug)]
 pub struct ReflMemData {
@@ -36,29 +33,29 @@ pub enum ReflMemKind {
 pub struct ReflModData {
 	inferred: Mutex<bool>,
 	path: VPath,
-	members: MemoMap<IStr, ReflMem>,
+	members: MemoMap<Tok<String>, ReflMem>,
 }
 
 #[derive(Clone, Debug)]
 pub struct ReflMod(Rc<ReflModData>);
 impl ReflMod {
-	pub fn path(&self) -> &[IStr] { &self.0.path[..] }
+	pub fn path(&self) -> &[Tok<String>] { &self.0.path[..] }
 	pub fn is_root(&self) -> bool { self.0.path.is_empty() }
 	async fn try_populate(&self) -> Result<(), api::LsModuleError> {
-		let path_tok = iv(&self.0.path[..]).await;
+		let path_tok = i().i(&self.0.path[..]).await;
-		let reply = match request(api::LsModule(sys_id(), path_tok.to_api())).await {
+		let reply = match ctx().reqnot().request(api::LsModule(ctx().sys_id(), path_tok.to_api())).await
-			Err(api::LsModuleError::TreeUnavailable) => {
+		{
-				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty.")
+			Err(api::LsModuleError::TreeUnavailable) =>
-			},
+				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty."),
 			Err(err) => return Err(err),
 			Ok(details) => details,
 		};
 		for (k, v) in reply.members {
-			let k = es(k).await;
+			let k = i().ex(k).await;
 			let mem = match self.0.members.get(&k) {
 				Some(mem) => mem,
 				None => {
-					let path = self.0.path.clone().name_with_suffix(k.clone()).to_sym().await;
+					let path = self.0.path.clone().name_with_suffix(k.clone()).to_sym(&i()).await;
 					let kind = match v.kind {
 						api::MemberInfoKind::Constant => ReflMemKind::Const,
 						api::MemberInfoKind::Module =>
@@ -71,7 +68,7 @@ impl ReflMod {
 		}
 		Ok(())
 	}
-	pub async fn get_child(&self, key: &IStr) -> Option<ReflMem> {
+	pub async fn get_child(&self, key: &Tok<String>) -> Option<ReflMem> {
 		let inferred_g = self.0.inferred.lock().await;
 		if let Some(mem) = self.0.members.get(key) {
 			return Some(mem.clone());
@@ -80,18 +77,16 @@ impl ReflMod {
 			return None;
 		}
 		match self.try_populate().await {
-			Err(api::LsModuleError::InvalidPath) => {
+			Err(api::LsModuleError::InvalidPath) =>
-				panic!("Path became invalid since module was created")
+				panic!("Path became invalid since module was created"),
-			},
+			Err(api::LsModuleError::IsConstant) =>
-			Err(api::LsModuleError::IsConstant) => {
+				panic!("Path previously contained a module but now contains a constant"),
-				panic!("Path previously contained a module but now contains a constant")
-			},
 			Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
 			Ok(()) => (),
 		}
 		self.0.members.get(key).cloned()
 	}
-	pub async fn get_by_path(&self, path: &[IStr]) -> Result<ReflMem, InvalidPathError> {
+	pub async fn get_by_path(&self, path: &[Tok<String>]) -> Result<ReflMem, InvalidPathError> {
 		let (next, tail) = path.split_first().expect("Attempted to walk by empty path");
 		let inferred_g = self.0.inferred.lock().await;
 		if let Some(next) = self.0.members.get(next) {
@@ -135,9 +130,9 @@ impl ReflMod {
 	}
 }
 
-task_local! {
+#[derive(Clone)]
-	static REFL_ROOTS: RefCell<HashMap<api::SysId, ReflMod>>
+struct ReflRoot(ReflMod);
-}
+impl SysCtxEntry for ReflRoot {}
 
 #[derive(Clone, Debug)]
 pub struct InvalidPathError {
@@ -155,12 +150,8 @@ fn default_member(is_root: bool, kind: ReflMemKind) -> ReflMem {
 	}))
 }
 
-pub fn refl() -> ReflMod {
+fn get_root() -> ReflRoot {
-	REFL_ROOTS.with(|tbl| {
+	ctx().get_or_insert(|| ReflRoot(default_module(VPath::new([])))).clone()
-		tbl.borrow_mut().entry(sys_id()).or_insert_with(|| default_module(VPath::new([]))).clone()
-	})
 }
 
-pub fn with_refl_roots<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+pub fn refl() -> ReflMod { get_root().0.clone() }
-	Box::pin(REFL_ROOTS.scope(RefCell::default(), fut))
-}

orchid-extension/src/stream_reqs.rs

@@ -1,72 +0,0 @@
-use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::Request;
-
-use crate::atom::AtomMethod;
-
-/// Represents [std::io::ErrorKind] values that are produced while operating on
-/// already-opened files
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
-pub enum IoErrorKind {
-	BrokenPipe,
-	UnexpectedEof,
-	ConnectionAborted,
-	Other,
-}
-
-/// Represents [std::io::Error] values that are produced while operating on
-/// already-opened files
-#[derive(Clone, Debug, Coding)]
-pub struct IoError {
-	pub message: String,
-	pub kind: IoErrorKind,
-}
-
-/// Read at most the specified number of bytes, but at least one byte, from a
-/// stream. If the returned vector is empty, the stream has reached its end.
-#[derive(Clone, Debug, Coding, Hierarchy)]
-pub struct ReadReq(pub u64);
-impl Request for ReadReq {
-	type Response = Result<Vec<u8>, IoError>;
-}
-impl AtomMethod for ReadReq {
-	const NAME: &str = "orchid::stream::read";
-}
-
-/// Write the specified number of bytes into a stream.
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(OutputReq)]
-pub struct WriteReq {
-	pub data: Vec<u8>,
-}
-impl Request for WriteReq {
-	type Response = Result<(), IoError>;
-}
-
-/// Flush a stream, ensuring that all data reached its destination.
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(OutputReq)]
-pub struct FlushReq;
-impl Request for FlushReq {
-	type Response = Result<(), IoError>;
-}
-
-/// Close a stream, indicating that no further data will be sent through it.
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(OutputReq)]
-pub struct CloseReq;
-impl Request for CloseReq {
-	type Response = Result<(), IoError>;
-}
-
-/// Operations on outbound streams across extension boundaries.
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extendable]
-#[allow(clippy::enum_variant_names)]
-pub enum OutputReq {
-	WriteReq(WriteReq),
-	FlushReq(FlushReq),
-	CloseReq(CloseReq),
-}
-impl AtomMethod for OutputReq {
-	const NAME: &str = "orchid::stream::write";
-}

orchid-extension/src/system.rs

@@ -1,5 +1,4 @@
 use std::any::{Any, TypeId};
-use std::fmt::Debug;
 use std::future::Future;
 use std::num::NonZero;
 use std::pin::Pin;
@@ -9,13 +8,13 @@ use futures::future::LocalBoxFuture;
 use orchid_api_traits::{Coding, Decode, Encode, Request};
 use orchid_base::boxed_iter::BoxedIter;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::{Receipt, ReqHandle, ReqReader, ReqReaderExt};
+use orchid_base::reqnot::{Receipt, Requester};
-use task_local::task_local;
 
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TAtom, get_info};
+use crate::context::ctx;
 use crate::coroutine_exec::Replier;
-use crate::entrypoint::request;
+use crate::entrypoint::ExtReq;
 use crate::func_atom::{Fun, Lambda};
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
@@ -23,7 +22,7 @@ use crate::system_ctor::{CtedObj, SystemCtor};
 use crate::tree::GenMember;
 
 /// System as consumed by foreign code
-pub trait SystemCard: Debug + Default + Send + Sync + 'static {
+pub trait SystemCard: Default + Send + Sync + 'static {
 	type Ctor: SystemCtor;
 	type Req: Coding;
 	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>>;
@@ -68,7 +67,7 @@ pub async fn resolv_atom(
 	sys: &(impl DynSystemCard + ?Sized),
 	atom: &api::Atom,
 ) -> Box<dyn AtomDynfo> {
-	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data.0[..])).await.unwrap();
+	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data.0[..])).await;
 	atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
@@ -85,10 +84,7 @@ pub trait System: Send + Sync + SystemCard + 'static {
 	fn env() -> impl Future<Output = Vec<GenMember>>;
 	fn lexers() -> Vec<LexerObj>;
 	fn parsers() -> Vec<ParserObj>;
-	fn request<'a>(
+	fn request(hand: ExtReq<'_>, req: Self::Req) -> impl Future<Output = Receipt<'_>>;
-		hand: Box<dyn ReqHandle<'a> + 'a>,
-		req: Self::Req,
-	) -> impl Future<Output = Receipt<'a>>;
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
@@ -96,7 +92,7 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
 	fn dyn_env(&self) -> LocalBoxFuture<'_, Vec<GenMember>>;
 	fn dyn_lexers(&self) -> Vec<LexerObj>;
 	fn dyn_parsers(&self) -> Vec<ParserObj>;
-	fn dyn_request<'a>(&self, hand: Box<dyn ReqReader<'a> + 'a>) -> LocalBoxFuture<'a, Receipt<'a>>;
+	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>>;
 	fn card(&self) -> &dyn DynSystemCard;
 }
 
@@ -105,41 +101,26 @@ impl<T: System> DynSystem for T {
 	fn dyn_env(&self) -> LocalBoxFuture<'_, Vec<GenMember>> { Self::env().boxed_local() }
 	fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
 	fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
-	fn dyn_request<'a>(
+	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>> {
-		&self,
-		mut hand: Box<dyn ReqReader<'a> + 'a>,
-	) -> LocalBoxFuture<'a, Receipt<'a>> {
 		Box::pin(async move {
-			let value = hand.read_req::<<Self as SystemCard>::Req>().await.unwrap();
+			Self::request(hand, <Self as SystemCard>::Req::decode(Pin::new(&mut &req[..])).await).await
-			Self::request(hand.finish().await, value).await
 		})
 	}
 	fn card(&self) -> &dyn DynSystemCard { self }
 }
 
-#[derive(Clone)]
-pub(crate) struct SysCtx(pub api::SysId, pub CtedObj);
-
-task_local! {
-	static SYS_CTX: SysCtx;
-}
-
-pub(crate) async fn with_sys<F: Future>(sys: SysCtx, fut: F) -> F::Output {
-	SYS_CTX.scope(sys, fut).await
-}
-
-pub fn sys_id() -> api::SysId { SYS_CTX.with(|cx| cx.0) }
-pub fn cted() -> CtedObj { SYS_CTX.with(|cx| cx.1.clone()) }
 pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TAtom<A>, ForeignAtom>
 where A: AtomicFeatures {
 	let mut data = &foreign.atom.data.0[..];
-	let value = AtomTypeId::decode_slice(&mut data);
+	let ctx = ctx();
-	let cted = cted();
+	let value = AtomTypeId::decode(Pin::new(&mut data)).await;
-	let own_inst = cted.inst();
+	let own_inst = ctx.get::<CtedObj>().inst();
-	let owner = if sys_id() == foreign.atom.owner {
+	let owner = if *ctx.get::<api::SysId>() == foreign.atom.owner {
 		own_inst.card()
 	} else {
-		cted.deps().find(|s| s.id() == foreign.atom.owner).ok_or_else(|| foreign.clone())?.get_card()
+		(ctx.get::<CtedObj>().deps().find(|s| s.id() == foreign.atom.owner))
+			.ok_or_else(|| foreign.clone())?
+			.get_card()
 	};
 	if owner.atoms().flatten().all(|dynfo| dynfo.tid() != TypeId::of::<A>()) {
 		return Err(foreign);
@@ -149,27 +130,22 @@ where A: AtomicFeatures {
 		return Err(foreign);
 	}
 	let val = dynfo.decode(AtomCtx(data, foreign.atom.drop)).await;
-	let Ok(value) = val.downcast::<A::Data>() else {
+	let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
-		panic!("decode of {} returned wrong type.", dynfo.name());
+	Ok(TAtom { value, untyped: foreign })
-	};
-	Ok(TAtom { value: *value, untyped: foreign })
 }
 
-/// Make a global request to a system that supports this request type. The
+pub async fn dep_req<Sys: SystemCard, Req: Request + Into<Sys::Req>>(req: Req) -> Req::Response {
-/// target system must either be the system in which this function is called, or
+	let ctx = ctx();
-/// one of its direct dependencies.
-pub async fn sys_req<Sys: SystemCard, Req: Request + Into<Sys::Req>>(req: Req) -> Req::Response {
 	let mut msg = Vec::new();
-	req.into().encode_vec(&mut msg);
+	req.into().encode(std::pin::pin!(&mut msg)).await;
-	let cted = cted();
+	let own_inst = ctx.get::<CtedObj>().inst();
-	let own_inst = cted.inst();
 	let owner = if own_inst.card().type_id() == TypeId::of::<Sys>() {
-		sys_id()
+		ctx.sys_id()
 	} else {
-		(cted.deps().find(|s| s.get_card().type_id() == TypeId::of::<Sys>()))
+		(ctx.get::<CtedObj>().deps().find(|s| s.get_card().type_id() == TypeId::of::<Sys>()))
 			.expect("System not in dependency array")
 			.id()
 	};
-	let reply = request(api::SysFwd(owner, msg)).await;
+	let reply = ctx.reqnot().request(api::SysFwd(owner, msg)).await;
-	Req::Response::decode(std::pin::pin!(&reply[..])).await.unwrap()
+	Req::Response::decode(std::pin::pin!(&reply[..])).await
 }

orchid-extension/src/system_ctor.rs

@@ -1,5 +1,4 @@
 use std::any::Any;
-use std::fmt::Debug;
 use std::sync::Arc;
 
 use orchid_base::boxed_iter::{BoxedIter, box_empty, box_once};
@@ -9,7 +8,6 @@ use crate::api;
 use crate::other_system::{DynSystemHandle, SystemHandle};
 use crate::system::{DynSystem, System, SystemCard};
 
-#[derive(Debug)]
 pub struct Cted<Ctor: SystemCtor + ?Sized> {
 	pub deps: <Ctor::Deps as DepDef>::Sat,
 	pub inst: Arc<Ctor::Instance>,
@@ -17,7 +15,7 @@ pub struct Cted<Ctor: SystemCtor + ?Sized> {
 impl<C: SystemCtor + ?Sized> Clone for Cted<C> {
 	fn clone(&self) -> Self { Self { deps: self.deps.clone(), inst: self.inst.clone() } }
 }
-pub trait DynCted: Debug + Send + Sync + 'static {
+pub trait DynCted: Send + Sync + 'static {
 	fn as_any(&self) -> &dyn Any;
 	fn deps<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)>;
 	fn inst(&self) -> Arc<dyn DynSystem>;
@@ -29,11 +27,11 @@ impl<C: SystemCtor + ?Sized> DynCted for Cted<C> {
 }
 pub type CtedObj = Arc<dyn DynCted>;
 
-pub trait DepSat: Debug + Clone + Send + Sync + 'static {
+pub trait DepSat: Clone + Send + Sync + 'static {
 	fn iter<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)>;
 }
 
-pub trait DepDef: Debug {
+pub trait DepDef {
 	type Sat: DepSat;
 	fn report(names: &mut impl FnMut(&'static str));
 	fn create(take: &mut impl FnMut() -> api::SysId) -> Self::Sat;
@@ -59,16 +57,17 @@ impl DepDef for () {
 	fn report(_: &mut impl FnMut(&'static str)) {}
 }
 
-pub trait SystemCtor: Debug + Send + Sync + 'static {
+pub trait SystemCtor: Send + Sync + 'static {
 	type Deps: DepDef;
 	type Instance: System;
 	const NAME: &'static str;
 	const VERSION: f64;
-	/// Create a system instance.
+	/// Create a system instance. When this function is called, a context object
-	fn inst(&self, deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
+	/// isn't yet available
+	fn inst(deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
 }
 
-pub trait DynSystemCtor: Debug + Send + Sync + 'static {
+pub trait DynSystemCtor: Send + Sync + 'static {
 	fn decl(&self, id: api::SysDeclId) -> api::SystemDecl;
 	fn new_system(&self, new: &api::NewSystem) -> CtedObj;
 }
@@ -85,7 +84,7 @@ impl<T: SystemCtor> DynSystemCtor for T {
 	fn new_system(&self, api::NewSystem { system: _, id: _, depends }: &api::NewSystem) -> CtedObj {
 		let mut ids = depends.iter().copied();
 		let deps = T::Deps::create(&mut || ids.next().unwrap());
-		let inst = Arc::new(self.inst(deps.clone()));
+		let inst = Arc::new(T::inst(deps.clone()));
 		Arc::new(Cted::<T> { deps, inst })
 	}
 }
@@ -152,4 +151,8 @@ mod dep_set_tuple_impls {
 	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J);
 	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K);
 	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L); // 12
+	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M);
+	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M, N);
+	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O);
+	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P); // 16
 }

orchid-extension/src/tokio.rs

@@ -1,38 +1,57 @@
+use crate::entrypoint::ExtensionData;
+
+#[cfg(feature = "tokio")]
+pub async fn tokio_main(data: ExtensionData) {
+	use std::io::{ErrorKind, Write};
+	use std::mem;
+	use std::pin::{Pin, pin};
 	use std::rc::Rc;
 
-use crate::entrypoint::ExtensionBuilder;
+	use async_once_cell::OnceCell;
-use crate::ext_port::ExtPort;
+	use futures::StreamExt;
-/// Run an extension inside a Tokio localset. Since the extension API does not
+	use futures::future::LocalBoxFuture;
-/// provide a forking mechanism, it can safely abort once the localset is
+	use futures::lock::Mutex;
-/// exhausted. If an extension absolutely needs a parallel thread, it can import
+	use futures::stream::FuturesUnordered;
-/// and call [tokio::task::spawn_local] which will keep alive the localset and
+	use orchid_api_traits::{Decode, Encode};
-/// postpone the aggressive shutdown, and listen for the [Drop::drop] of the
+	use orchid_base::msg::{recv_msg, send_msg};
-/// value returned by [crate::system_ctor::SystemCtor::inst] to initiate
+	use tokio::io::{Stdout, stdin, stdout};
-/// shutdown.
-#[cfg(feature = "tokio")]
-pub async fn tokio_entrypoint(builder: ExtensionBuilder) {
-	use tokio::io::{stderr, stdin, stdout};
 	use tokio::task::{LocalSet, spawn_local};
-	use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
+	use tokio_util::compat::{Compat, TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
+
+	use crate::api;
+	use crate::entrypoint::extension_init;
 
 	let local_set = LocalSet::new();
 	local_set.spawn_local(async {
-		builder.build(ExtPort {
+		let host_header = api::HostHeader::decode(Pin::new(&mut stdin().compat())).await;
-			input: Box::pin(stdin().compat()),
+		let init =
-			output: Box::pin(stdout().compat_write()),
+			Rc::new(extension_init(data, host_header, Rc::new(|fut| mem::drop(spawn_local(fut)))));
-			log: Box::pin(stderr().compat_write()),
+		let mut buf = Vec::new();
-			spawn: Rc::new(|fut| {
+		init.header.encode(Pin::new(&mut buf)).await;
-				spawn_local(fut);
+		std::io::stdout().write_all(&buf).unwrap();
-			}),
+		std::io::stdout().flush().unwrap();
-		});
+		// These are concurrent processes that never exit, so if the FuturesUnordered
+		// produces any result the extension should exit
+		let mut io = FuturesUnordered::<LocalBoxFuture<()>>::new();
+		io.push(Box::pin(async {
+			loop {
+				match recv_msg(pin!(stdin().compat())).await {
+					Ok(msg) => init.send(&msg[..]).await,
+					Err(e) if e.kind() == ErrorKind::BrokenPipe => break,
+					Err(e) if e.kind() == ErrorKind::UnexpectedEof => break,
+					Err(e) => panic!("{e}"),
+				}
+			}
+		}));
+		io.push(Box::pin(async {
+			while let Some(msg) = init.recv().await {
+				static STDOUT: OnceCell<Mutex<Compat<Stdout>>> = OnceCell::new();
+				let stdout_lk = STDOUT.get_or_init(async { Mutex::new(stdout().compat_write()) }).await;
+				let mut stdout_g = stdout_lk.lock().await;
+				send_msg(pin!(&mut *stdout_g), &msg[..]).await.expect("Parent pipe broken");
+			}
+		}));
+		io.next().await;
 	});
 	local_set.await;
 }
-
-#[macro_export]
-macro_rules! tokio_main {
-	($builder:expr) => {
-		#[tokio::main]
-		pub async fn main() { $crate::tokio::tokio_entrypoint($builder).await }
-	};
-}

orchid-extension/src/tree.rs

@@ -1,6 +1,4 @@
-use std::cell::RefCell;
 use std::num::NonZero;
-use std::rc::Rc;
 
 use async_fn_stream::stream;
 use dyn_clone::{DynClone, clone_box};
@@ -8,19 +6,20 @@ use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::interner::{IStr, is};
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::tree::{TokTree, Token, TokenVariant};
 use substack::Substack;
-use task_local::task_local;
 use trait_set::trait_set;
 
 use crate::api;
+use crate::context::i;
 use crate::conv::ToExpr;
+use crate::entrypoint::MemberRecord;
 use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
 use crate::func_atom::{ExprFunc, Fun};
-use crate::gen_expr::{GExpr, new_atom, sym_ref};
+use crate::gen_expr::{GExpr, sym_ref};
 
 pub type GenTokTree = TokTree<Expr, GExpr>;
 pub type GenTok = Token<Expr, GExpr>;
@@ -28,7 +27,12 @@ pub type GenTok = Token<Expr, GExpr>;
 impl TokenVariant<api::Expression> for GExpr {
 	type FromApiCtx<'a> = ();
 	type ToApiCtx<'a> = ();
-	async fn from_api(_: &api::Expression, _: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+	async fn from_api(
+		_: &api::Expression,
+		_: &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
 		panic!("Received new expression from host")
 	}
 	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> api::Expression { self.serialize().await }
@@ -36,7 +40,12 @@ impl TokenVariant<api::Expression> for GExpr {
 
 impl TokenVariant<api::ExprTicket> for Expr {
 	type FromApiCtx<'a> = &'a BorrowedExprStore;
-	async fn from_api(api: &api::ExprTicket, exprs: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+	async fn from_api(
+		api: &api::ExprTicket,
+		exprs: &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
 		// SAFETY: receiving trees from sublexers implies borrowing
 		Expr::from_handle(ExprHandle::borrowed(*api, exprs))
 	}
@@ -71,11 +80,13 @@ pub fn module(
 	vec![GenMember { name, kind, public, comments: vec![] }]
 }
 pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (String, MemKind) {
-	(name.to_string(), MemKind::module(mems))
+	let kind = MemKind::Mod { members: mems.into_iter().flatten().collect() };
+	(name.to_string(), kind)
 }
 pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
-	let fac =
+	let fac = LazyMemberFactory::new(async move |sym| {
-		LazyMemberFactory::new(async move |sym| MemKind::Const(new_atom(Fun::new(sym, xf).await)));
+		MemKind::Const(Fun::new(sym, xf).await.to_gen().await)
+	});
 	vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
 }
 pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
@@ -112,12 +123,12 @@ pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<Gen
 				let prev = all_members.insert(mem.name.clone(), (unit, mem.comments.into_iter().collect()));
 				assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name);
 			},
-			MemKind::Mod(members) => match all_members.entry(mem.name.clone()) {
+			MemKind::Mod { members } => match all_members.entry(mem.name.clone()) {
 				hashbrown::hash_map::Entry::Vacant(slot) => {
-					slot.insert((MemKind::Mod(members), mem.comments.into_iter().collect()));
+					slot.insert((MemKind::Mod { members }, mem.comments.into_iter().collect()));
 				},
 				hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() {
-					(MemKind::Mod(old_items), old_cmts) => {
+					(MemKind::Mod { members: old_items, .. }, old_cmts) => {
 						let mut swap = vec![];
 						std::mem::swap(&mut swap, old_items);
 						*old_items = merge_trivial([swap, members]);
@@ -156,29 +167,25 @@ pub struct GenMember {
 	pub comments: Vec<String>,
 }
 impl GenMember {
-	pub(crate) async fn into_api(self, tia_cx: &mut impl TreeIntoApiCtx) -> api::Member {
+	pub async fn into_api(self, tia_cx: &mut impl TreeIntoApiCtx) -> api::Member {
-		let name = is(&self.name).await;
+		let name = i().i::<String>(&self.name).await;
 		let kind = self.kind.into_api(&mut tia_cx.push_path(name.clone())).await;
-		let comments = join_all(self.comments.iter().map(async |cmt| is(cmt).await.to_api())).await;
+		let comments = join_all(self.comments.iter().map(async |cmt| i().i(cmt).await.to_api())).await;
 		api::Member { kind, name: name.to_api(), comments, exported: self.public }
 	}
 }
 
 pub enum MemKind {
 	Const(GExpr),
-	Mod(Vec<GenMember>),
+	Mod { members: Vec<GenMember> },
 	Lazy(LazyMemberFactory),
 }
 impl MemKind {
-	pub async fn cnst(val: impl ToExpr) -> Self { Self::Const(val.to_gen().await) }
+	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
-	pub fn module(mems: impl IntoIterator<Item = Vec<GenMember>>) -> Self {
-		Self::Mod(mems.into_iter().flatten().collect())
-	}
-	pub(crate) async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
 		match self {
-			Self::Lazy(lazy) => api::MemberKind::Lazy(add_lazy(ctx, lazy)),
+			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
 			Self::Const(c) => api::MemberKind::Const(c.serialize().await),
-			Self::Mod(members) => api::MemberKind::Module(api::Module {
+			Self::Mod { members } => api::MemberKind::Module(api::Module {
 				members: stream(async |mut cx| {
 					for m in members {
 						cx.emit(m.into_api(ctx).await).await
@@ -192,58 +199,29 @@ impl MemKind {
 	}
 }
 
-pub enum MemberRecord {
+pub trait TreeIntoApiCtx {
-	Gen(Vec<IStr>, LazyMemberFactory),
+	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
-	Res,
+	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
 }
 
-#[derive(Clone, Default)]
+pub struct TreeIntoApiCtxImpl<'a, 'b> {
-pub(crate) struct LazyMemberStore(Rc<RefCell<HashMap<api::TreeId, MemberRecord>>>);
+	pub basepath: &'a [Tok<String>],
-
+	pub path: Substack<'a, Tok<String>>,
-task_local! {
+	pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
-	static LAZY_MEMBERS: LazyMemberStore;
 }
 
-pub fn with_lazy_member_store<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+impl TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_> {
-	Box::pin(LAZY_MEMBERS.scope(LazyMemberStore::default(), fut))
+	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
+		TreeIntoApiCtxImpl {
+			lazy_members: self.lazy_members,
+			basepath: self.basepath,
+			path: self.path.push(seg),
 		}
-
+	}
-fn add_lazy(cx: &impl TreeIntoApiCtx, fac: LazyMemberFactory) -> api::TreeId {
+	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId {
-	LAZY_MEMBERS.with(|lazy_members| {
+		let id = api::TreeId(NonZero::new((self.lazy_members.len() + 2) as u64).unwrap());
-		let mut g = lazy_members.0.borrow_mut();
+		let path = self.basepath.iter().cloned().chain(self.path.unreverse()).collect_vec();
-		let id = api::TreeId(NonZero::new((g.len() + 2) as u64).unwrap());
+		self.lazy_members.insert(id, MemberRecord::Gen(path, fac));
-		let path = cx.path().collect_vec();
-		g.insert(id, MemberRecord::Gen(path, fac));
 		id
-	})
-}
-
-pub async fn get_lazy(id: api::TreeId) -> (Sym, MemKind) {
-	let (path, cb) =
-		LAZY_MEMBERS.with(|tbl| match tbl.0.borrow_mut().insert(id, MemberRecord::Res) {
-			None => panic!("Tree for ID not found"),
-			Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
-			Some(MemberRecord::Gen(path, cb)) => (path, cb),
-		});
-	let path = Sym::new(path).await.unwrap();
-	(path.clone(), cb.build(path).await)
-}
-
-pub(crate) trait TreeIntoApiCtx {
-	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx;
-	fn path(&self) -> impl Iterator<Item = IStr>;
-}
-
-pub struct TreeIntoApiCtxImpl<'a> {
-	pub basepath: &'a [IStr],
-	pub path: Substack<'a, IStr>,
-}
-
-impl TreeIntoApiCtx for TreeIntoApiCtxImpl<'_> {
-	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx {
-		TreeIntoApiCtxImpl { basepath: self.basepath, path: self.path.push(seg) }
-	}
-	fn path(&self) -> impl Iterator<Item = IStr> {
-		self.basepath.iter().cloned().chain(self.path.unreverse())
 	}
 }

orchid-extension/src/trivial_req.rs

@@ -1,28 +0,0 @@
-use std::io;
-use std::pin::Pin;
-
-use futures::future::LocalBoxFuture;
-use futures::{AsyncRead, AsyncWrite};
-use orchid_base::reqnot::{Receipt, RepWriter, ReqHandle, ReqReader};
-
-pub struct TrivialReqCycle<'a> {
-	pub req: &'a [u8],
-	pub rep: &'a mut Vec<u8>,
-}
-impl<'a> ReqReader<'a> for TrivialReqCycle<'a> {
-	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { Pin::new(&mut self.req) as Pin<&mut _> }
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>> {
-		Box::pin(async { self as Box<_> })
-	}
-}
-impl<'a> ReqHandle<'a> for TrivialReqCycle<'a> {
-	fn start_reply(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>> {
-		Box::pin(async { Ok(self as Box<_>) })
-	}
-}
-impl<'a> RepWriter<'a> for TrivialReqCycle<'a> {
-	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { Pin::new(&mut self.rep) as Pin<&mut _> }
-	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Receipt<'a>>> {
-		Box::pin(async { Ok(Receipt::_new()) })
-	}
-}

orchid-host/Cargo.toml

@@ -8,29 +8,22 @@ edition = "2024"
 [dependencies]
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
+async-process = "2.4.0"
 bound = "0.6.0"
 derive_destructure = "1.0.0"
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 futures-locks = "0.7.1"
-hashbrown = "0.16.1"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
-libloading = { version = "0.9.0", optional = true }
 memo-map = "0.3.3"
 never = "0.1.0"
 num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-orchid-extension = { version = "0.1.0", path = "../orchid-extension", optional = true }
+ordered-float = "5.0.0"
-ordered-float = "5.1.0"
+pastey = "0.1.1"
-pastey = "0.2.1"
 substack = "1.1.1"
-tokio = { version = "1.49.0", features = ["process"], optional = true }
+test_executors = "0.3.5"
-tokio-util = { version = "0.7.18", features = ["compat"], optional = true }
 trait-set = "0.3.0"
-unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
-
-[features]
-tokio = ["dep:tokio", "dep:tokio-util", "dep:libloading"]
-orchid-extension = ["dep:orchid-extension"]

orchid-host/src/atom.rs

@@ -3,14 +3,10 @@ use std::rc::{Rc, Weak};
 
 use async_once_cell::OnceCell;
 use derive_destructure::destructure;
-#[cfg(feature = "orchid-extension")]
-use orchid_api_traits::{Request, UnderRoot};
 use orchid_base::format::{FmtCtx, FmtUnit, Format, take_first_fmt};
 use orchid_base::location::Pos;
-use orchid_base::reqnot::ClientExt;
+use orchid_base::reqnot::Requester;
 use orchid_base::tree::AtomRepr;
-#[cfg(feature = "orchid-extension")]
-use orchid_extension::atom::AtomMethod;
 
 use crate::api;
 use crate::ctx::Ctx;
@@ -60,29 +56,14 @@ impl AtomHand {
 		Self(Rc::new(AtomData { owner, drop, data, display: OnceCell::new() }))
 	}
 	#[must_use]
-	#[cfg(feature = "orchid-extension")]
-	pub async fn ipc<M: Request + UnderRoot<Root: AtomMethod>>(
-		&self,
-		method: M,
-	) -> Option<M::Response> {
-		use orchid_api_traits::{Decode, Encode};
-		use orchid_base::name::Sym;
-
-		let name = Sym::parse(<M as UnderRoot>::Root::NAME).await.unwrap();
-		let mut buf = Vec::new();
-		method.into_root().encode_vec(&mut buf);
-		let reply_buf = self.req(name.to_api(), buf).await?;
-		Some(M::Response::decode_slice(&mut &reply_buf[..]))
-	}
-	#[must_use]
 	pub async fn call(self, arg: Expr) -> Expr {
 		let owner_sys = self.0.owner.clone();
 		let ctx = owner_sys.ctx();
-		let client = owner_sys.client();
+		let reqnot = owner_sys.reqnot();
 		ctx.exprs.give_expr(arg.clone());
 		let ret = match Rc::try_unwrap(self.0) {
-			Ok(data) => client.request(api::FinalCall(data.api(), arg.id())).await.unwrap(),
+			Ok(data) => reqnot.request(api::FinalCall(data.api(), arg.id())).await,
-			Err(hand) => client.request(api::CallRef(hand.api_ref(), arg.id())).await.unwrap(),
+			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
 		};
 		let val = Expr::from_api(&ret, PathSetBuilder::new(), ctx.clone()).await;
 		ctx.exprs.take_expr(arg.id());
@@ -93,21 +74,19 @@ impl AtomHand {
 	#[must_use]
 	pub fn ext(&self) -> &Extension { self.sys().ext() }
 	pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
-		self.0.owner.client().request(api::Fwded(self.0.api_ref(), key, req)).await.unwrap()
+		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
 	}
 	#[must_use]
 	pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
 	#[must_use]
-	pub async fn to_string(&self) -> String { take_first_fmt(self).await }
+	pub async fn to_string(&self) -> String { take_first_fmt(self, &self.0.owner.ctx().i).await }
 	#[must_use]
 	pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) }
 }
 impl Format for AtomHand {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		(self.0.display.get_or_init(async {
-			FmtUnit::from_api(
+			FmtUnit::from_api(&self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await)
-				&self.0.owner.client().request(api::AtomPrint(self.0.api_ref())).await.unwrap(),
-			)
 		}))
 		.await
 		.clone()

orchid-host/src/ctx.rs

@@ -3,32 +3,23 @@ use std::num::{NonZero, NonZeroU16};
 use std::rc::{Rc, Weak};
 use std::{fmt, ops};
 
-use futures::future::LocalBoxFuture;
 use futures_locks::RwLock;
 use hashbrown::HashMap;
+use orchid_base::builtin::Spawner;
+use orchid_base::interner::Interner;
 
 use crate::api;
 use crate::expr_store::ExprStore;
-use crate::logger::LoggerImpl;
 use crate::system::{System, WeakSystem};
 use crate::tree::WeakRoot;
 
-pub trait JoinHandle {
-	fn abort(&self);
-	fn join(self: Box<Self>) -> LocalBoxFuture<'static, ()>;
-}
-
-pub trait Spawner {
-	fn spawn_obj(&self, fut: LocalBoxFuture<'static, ()>) -> Box<dyn JoinHandle>;
-}
-
 pub struct CtxData {
-	spawner: Rc<dyn Spawner>,
+	pub i: Interner,
+	pub spawn: Spawner,
 	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
 	pub system_id: RefCell<NonZeroU16>,
 	pub exprs: ExprStore,
 	pub root: RwLock<WeakRoot>,
-	pub logger: LoggerImpl,
 }
 #[derive(Clone)]
 pub struct Ctx(Rc<CtxData>);
@@ -46,25 +37,16 @@ impl WeakCtx {
 }
 impl Ctx {
 	#[must_use]
-	pub fn new(spawner: impl Spawner + 'static, logger: LoggerImpl) -> Self {
+	pub fn new(spawn: Spawner) -> Self {
 		Self(Rc::new(CtxData {
-			spawner: Rc::new(spawner),
+			spawn,
+			i: Interner::default(),
 			systems: RwLock::default(),
 			system_id: RefCell::new(NonZero::new(1).unwrap()),
 			exprs: ExprStore::default(),
 			root: RwLock::default(),
-			logger,
 		}))
 	}
-	/// Spawn a parallel future that you can join at any later time.
-	///
-	/// Don't use this for async Drop, use [orchid_base::stash::stash] instead.
-	/// If you use this for an actor object, make sure to actually join the
-	/// handle.
-	#[must_use]
-	pub fn spawn(&self, fut: impl Future<Output = ()> + 'static) -> Box<dyn JoinHandle> {
-		self.spawner.spawn_obj(Box::pin(fut))
-	}
 	#[must_use]
 	pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {
 		self.systems.read().await.get(&id).and_then(WeakSystem::upgrade)
@@ -80,6 +62,9 @@ impl Ctx {
 }
 impl fmt::Debug for Ctx {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		f.debug_struct("Ctx").field("system_id", &self.system_id).finish_non_exhaustive()
+		f.debug_struct("Ctx")
+			.field("i", &self.i)
+			.field("system_id", &self.system_id)
+			.finish_non_exhaustive()
 	}
 }

orchid-host/src/dealias.rs

@@ -1,7 +1,7 @@
 use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_errv};
-use orchid_base::interner::{IStr, is};
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::Pos;
 use orchid_base::name::VName;
 
@@ -16,17 +16,17 @@ pub enum AbsPathError {
 	RootPath,
 }
 impl AbsPathError {
-	pub async fn err_obj(self, pos: Pos, path: &str) -> OrcErrv {
+	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErrv {
 		let (descr, msg) = match self {
 			AbsPathError::RootPath => (
-				is("Path ends on root module").await,
+				i.i("Path ends on root module").await,
 				format!(
 					"{path} is equal to the empty path. You cannot directly reference the root. \
 					Use one fewer 'super::' or add more segments to make it valid."
 				),
 			),
 			AbsPathError::TooManySupers => (
-				is("Too many 'super::' steps in path").await,
+				i.i("Too many 'super::' steps in path").await,
 				format!("{path} is leading outside the root."),
 			),
 		};
@@ -41,9 +41,13 @@ impl AbsPathError {
 ///
 /// if the relative path contains as many or more `super` segments than the
 /// length of the absolute path.
-pub async fn absolute_path(mut cwd: &[IStr], mut rel: &[IStr]) -> Result<VName, AbsPathError> {
+pub async fn absolute_path(
-	let i_self = is("self").await;
+	mut cwd: &[Tok<String>],
-	let i_super = is("super").await;
+	mut rel: &[Tok<String>],
+	i: &Interner,
+) -> Result<VName, AbsPathError> {
+	let i_self = i.i("self").await;
+	let i_super = i.i("super").await;
 	let mut relative = false;
 	if let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h == i_self) {
 		rel = tail;
@@ -59,13 +63,19 @@ pub async fn absolute_path(mut cwd: &[IStr], mut rel: &[IStr]) -> Result<VName,
 		.map_err(|_| AbsPathError::RootPath)
 }
 
+pub struct DealiasCtx<'a> {
+	pub i: &'a Interner,
+	pub rep: &'a Reporter,
+}
+
 pub async fn resolv_glob<Mod: Tree>(
-	cwd: &[IStr],
+	cwd: &[Tok<String>],
 	root: &Mod,
-	abs_path: &[IStr],
+	abs_path: &[Tok<String>],
 	pos: Pos,
+	i: &Interner,
 	ctx: &mut Mod::Ctx<'_>,
-) -> OrcRes<HashSet<IStr>> {
+) -> OrcRes<HashSet<Tok<String>>> {
 	let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
 	let (co_prefix, diff_path) = abs_path.split_at(abs_path.len().min(coprefix_len + 1));
 	let fst_diff =
@@ -79,7 +89,7 @@ pub async fn resolv_glob<Mod: Tree>(
 				ChildErrorKind::Missing => ("Invalid import path", format!("{path} not found")),
 				ChildErrorKind::Private => ("Import inaccessible", format!("{path} is private")),
 			};
-			return Err(mk_errv(is(tk).await, msg, [pos]));
+			return Err(mk_errv(i.i(tk).await, msg, [pos]));
 		},
 	};
 	Ok(target_module.children(coprefix_len < abs_path.len()))
@@ -90,11 +100,11 @@ pub type ChildResult<'a, T> = Result<&'a T, ChildErrorKind>;
 pub trait Tree {
 	type Ctx<'a>;
 	#[must_use]
-	fn children(&self, public_only: bool) -> HashSet<IStr>;
+	fn children(&self, public_only: bool) -> HashSet<Tok<String>>;
 	#[must_use]
 	fn child(
 		&self,
-		key: IStr,
+		key: Tok<String>,
 		public_only: bool,
 		ctx: &mut Self::Ctx<'_>,
 	) -> impl Future<Output = ChildResult<'_, Self>>;
@@ -125,7 +135,7 @@ pub struct ChildError {
 pub async fn walk<'a, T: Tree>(
 	root: &'a T,
 	public_only: bool,
-	path: impl IntoIterator<Item = IStr>,
+	path: impl IntoIterator<Item = Tok<String>>,
 	ctx: &mut T::Ctx<'_>,
 ) -> Result<&'a T, ChildError> {
 	let mut cur = root;

orchid-host/src/dylib.rs

@@ -1,59 +0,0 @@
-use std::io;
-use std::path::{Path, PathBuf};
-use std::sync::{Arc, Mutex};
-
-use futures::io::BufReader;
-use futures::{AsyncBufReadExt, StreamExt};
-use hashbrown::HashMap;
-use libloading::{Library, Symbol};
-use orchid_base::binary::vt_to_future;
-use orchid_base::logging::log;
-use unsync_pipe::pipe;
-
-use crate::api;
-use crate::ctx::Ctx;
-use crate::extension::ExtPort;
-
-static DYNAMIC_LIBRARIES: Mutex<Option<HashMap<PathBuf, Arc<Library>>>> = Mutex::new(None);
-fn load_dylib(path: &Path) -> Result<Arc<Library>, libloading::Error> {
-	let mut g = DYNAMIC_LIBRARIES.lock().unwrap();
-	let map = g.get_or_insert_default();
-	if let Some(lib) = map.get(path) {
-		Ok(lib.clone())
-	} else {
-		let lib = Arc::new(unsafe { Library::new(path) }?);
-		map.insert(path.to_owned(), lib.clone());
-		Ok(lib)
-	}
-}
-
-pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Error> {
-	let (write_input, input) = pipe(1024);
-	let (output, read_output) = pipe(1024);
-	let (write_log, read_log) = pipe(1024);
-	let log_path = path.to_string_lossy().to_string();
-	let _ = ctx.spawn(async move {
-		let mut lines = BufReader::new(read_log).lines();
-		while let Some(line) = lines.next().await {
-			match line {
-				Ok(line) => writeln!(log("stderr"), "dylib {log_path} err> {line}").await,
-				Err(e) => match e.kind() {
-					io::ErrorKind::BrokenPipe | io::ErrorKind::UnexpectedEof => break,
-					_ => panic!("Error while reading stderr {e}"),
-				},
-			}
-		}
-	});
-	let library = load_dylib(path)?;
-	let entrypoint: Symbol<unsafe extern "C" fn(api::binary::ExtensionContext)> =
-		unsafe { library.get("orchid_extension_main") }?;
-	let data = Box::into_raw(Box::new(ctx)) as *const ();
-	extern "C" fn drop(data: *const ()) { std::mem::drop(unsafe { Box::from_raw(data as *mut Ctx) }) }
-	extern "C" fn spawn(data: *const (), vt: api::binary::FutureBin) {
-		let _ = unsafe { (data as *mut Ctx).as_mut().unwrap().spawn(vt_to_future(vt)) };
-	}
-	let spawner = api::binary::SpawnerBin { data, drop, spawn };
-	let cx = api::binary::ExtensionContext { input, output, log: write_log, spawner };
-	unsafe { (entrypoint)(cx) };
-	Ok(ExtPort { input: Box::pin(write_input), output: Box::pin(read_output) })
-}

orchid-host/src/execute.rs

@@ -4,10 +4,11 @@ use bound::Bound;
 use futures::FutureExt;
 use futures_locks::{RwLockWriteGuard, TryLockError};
 use orchid_base::error::OrcErrv;
-use orchid_base::format::fmt;
+use orchid_base::format::{FmtCtxImpl, Format, take_first};
 use orchid_base::location::Pos;
-use orchid_base::logging::log;
+use orchid_base::logging::Logger;
 
+use crate::ctx::Ctx;
 use crate::expr::{Expr, ExprKind, PathSet, Step};
 use crate::tree::Root;
 
@@ -29,19 +30,21 @@ pub enum ExecResult {
 }
 
 pub struct ExecCtx {
+	ctx: Ctx,
 	gas: Option<u64>,
 	stack: Vec<ExprGuard>,
 	cur: ExprGuard,
 	cur_pos: Pos,
 	did_pop: bool,
+	logger: Logger,
 	root: Root,
 }
 impl ExecCtx {
 	#[must_use]
-	pub async fn new(root: Root, init: Expr) -> Self {
+	pub async fn new(ctx: Ctx, logger: Logger, root: Root, init: Expr) -> Self {
 		let cur_pos = init.pos();
 		let cur = Bound::async_new(init, |init| init.kind().write()).await;
-		Self { gas: None, stack: vec![], cur, cur_pos, did_pop: false, root }
+		Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger, root }
 	}
 	#[must_use]
 	pub fn remaining_gas(&self) -> u64 { self.gas.expect("queried remaining_gas but no gas was set") }
@@ -86,7 +89,8 @@ impl ExecCtx {
 		while self.use_gas(1) {
 			let mut kind_swap = ExprKind::Missing;
 			mem::swap(&mut kind_swap, &mut self.cur);
-			writeln!(log("debug"), "Exxecute lvl{} {}", self.stack.len(), fmt(&kind_swap).await).await;
+			let unit = kind_swap.print(&FmtCtxImpl { i: &self.ctx.i }).await;
+			writeln!(self.logger, "Exxecute lvl{} {}", self.stack.len(), take_first(&unit, true));
 			let (kind, op) = match kind_swap {
 				ExprKind::Identity(target) => {
 					let inner = self.unpack_ident(&target).await;
@@ -110,9 +114,8 @@ impl ExecCtx {
 						(ExprKind::Identity(val.clone()), StackOp::Swap(val))
 					},
 					Err(f) => match &*f.kind().read().await {
-						ExprKind::Arg | ExprKind::Call(..) | ExprKind::Seq(..) | ExprKind::Const(_) => {
+						ExprKind::Arg | ExprKind::Call(..) | ExprKind::Seq(..) | ExprKind::Const(_) =>
-							panic!("This should not appear outside function bodies")
+							panic!("This should not appear outside function bodies"),
-						},
 						ExprKind::Missing => panic!("Should have been replaced"),
 						ExprKind::Atom(a) => {
 							let x_norm = self.unpack_ident(&x).await;

orchid-host/src/expr.rs

@@ -9,6 +9,7 @@ use futures_locks::RwLock;
 use itertools::Itertools;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
+use orchid_base::interner::Interner;
 use orchid_base::location::{Pos, SrcRange};
 use orchid_base::name::Sym;
 use orchid_base::tl_cache;
@@ -55,13 +56,13 @@ impl Expr {
 	}
 	#[must_use]
 	pub async fn from_api(api: &api::Expression, psb: PathSetBuilder<'_, u64>, ctx: Ctx) -> Self {
-		let pos = Pos::from_api(&api.location).await;
+		let pos = Pos::from_api(&api.location, &ctx.i).await;
 		let kind = match &api.kind {
 			api::ExpressionKind::Arg(n) => {
 				assert!(psb.register_arg(n), "Arguments must be enclosed in a matching lambda");
 				ExprKind::Arg
 			},
-			api::ExpressionKind::Bottom(bot) => ExprKind::Bottom(OrcErrv::from_api(bot).await),
+			api::ExpressionKind::Bottom(bot) => ExprKind::Bottom(OrcErrv::from_api(bot, &ctx.i).await),
 			api::ExpressionKind::Call(f, x) => {
 				let (lpsb, rpsb) = psb.split();
 				ExprKind::Call(
@@ -69,7 +70,7 @@ impl Expr {
 					Expr::from_api(x, rpsb, ctx).boxed_local().await,
 				)
 			},
-			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name).await),
+			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name, &ctx.i).await),
 			api::ExpressionKind::Lambda(x, body) => {
 				let lbuilder = psb.lambda(x);
 				let body = Expr::from_api(body, lbuilder.stack(), ctx).boxed_local().await;
@@ -103,24 +104,18 @@ impl Expr {
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		return print_expr(self, c, Substack::Bottom, &[]).await;
+		return print_expr(self, c, Substack::Bottom).await;
 	}
 }
-pub async fn print_expr<'a>(
+async fn print_expr<'a>(
 	expr: &'a Expr,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
 	visited: Substack<'_, api::ExprTicket>,
-	id_only: &[api::ExprTicket],
 ) -> FmtUnit {
 	if visited.iter().any(|id| id == &expr.id()) {
 		return "CYCLIC_EXPR".to_string().into();
 	}
-	if id_only.iter().any(|id| id == &expr.id()) {
+	print_exprkind(&*expr.kind().read().await, c, visited.push(expr.id())).boxed_local().await
-		return format!("{:?}", expr.id()).into();
-	}
-	print_exprkind(&*expr.kind().read().await, c, visited.push(expr.id()), id_only)
-		.boxed_local()
-		.await
 }
 
 #[derive(Clone, Debug)]
@@ -144,30 +139,28 @@ impl ExprKind {
 }
 impl Format for ExprKind {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		print_exprkind(self, c, Substack::Bottom, &[]).await
+		print_exprkind(self, c, Substack::Bottom).await
 	}
 }
 async fn print_exprkind<'a>(
 	ek: &ExprKind,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
 	visited: Substack<'_, api::ExprTicket>,
-	id_only: &[api::ExprTicket],
 ) -> FmtUnit {
 	match &ek {
 		ExprKind::Arg => "Arg".to_string().into(),
-		ExprKind::Missing => {
+		ExprKind::Missing =>
-			panic!("This variant is swapped into write guards, so a read can never see it")
+			panic!("This variant is swapped into write guards, so a read can never see it"),
-		},
 		ExprKind::Atom(a) => a.print(c).await,
 		ExprKind::Bottom(e) if e.len() == 1 => format!("Bottom({e})").into(),
 		ExprKind::Bottom(e) => format!("Bottom(\n\t{}\n)", indent(&e.to_string())).into(),
 		ExprKind::Call(f, x) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
 			.unbounded("{0b} {1l}")
 			.bounded("({0b} {1})")))
-		.units([print_expr(f, c, visited, id_only).await, print_expr(x, c, visited, id_only).await]),
+		.units([print_expr(f, c, visited).await, print_expr(x, c, visited).await]),
 		ExprKind::Identity(id) =>
 			tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{{{0}}}"))).units([print_expr(
-				id, c, visited, id_only,
+				id, c, visited,
 			)
 			.boxed_local()
 			.await]),
@@ -175,14 +168,14 @@ async fn print_exprkind<'a>(
 		ExprKind::Lambda(None, body) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
 			// .unbounded("\\.{0l}")
 			.bounded("(\\.{0b})")))
-		.units([print_expr(body, c, visited, id_only).await]),
+		.units([print_expr(body, c, visited).await]),
 		ExprKind::Lambda(Some(path), body) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
 			// .unbounded("\\{0b}. {1l}")
 			.bounded("(\\{0b}. {1b})")))
-		.units([format!("{path}").into(), print_expr(body, c, visited, id_only).await]),
+		.units([format!("{path}").into(), print_expr(body, c, visited).await]),
 		ExprKind::Seq(l, r) =>
 			tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0b}]{1l}")))
-				.units([print_expr(l, c, visited, id_only).await, print_expr(r, c, visited, id_only).await]),
+				.units([print_expr(l, c, visited).await, print_expr(r, c, visited).await]),
 	}
 }
 
@@ -333,7 +326,12 @@ impl WeakExpr {
 
 impl TokenVariant<api::ExprTicket> for Expr {
 	type FromApiCtx<'a> = ExprStore;
-	async fn from_api(api: &api::ExprTicket, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+	async fn from_api(
+		api: &api::ExprTicket,
+		ctx: &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
 		ctx.get_expr(*api).expect("Invalid ticket")
 	}
 	type ToApiCtx<'a> = ExprStore;
@@ -350,7 +348,12 @@ pub struct ExprWillPanic;
 
 impl TokenVariant<api::Expression> for Expr {
 	type FromApiCtx<'a> = Ctx;
-	async fn from_api(api: &api::Expression, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+	async fn from_api(
+		api: &api::Expression,
+		ctx: &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
 		Self::from_api(api, PathSetBuilder::new(), ctx.clone()).await
 	}
 	type ToApiCtx<'a> = ExprWillPanic;

orchid-host/src/extension.rs

@@ -2,7 +2,7 @@ use std::cell::RefCell;
 use std::future::Future;
 use std::io;
 use std::num::NonZeroU64;
-use std::pin::Pin;
+use std::pin::pin;
 use std::rc::{Rc, Weak};
 
 use async_fn_stream::stream;
@@ -10,34 +10,28 @@ use derive_destructure::destructure;
 use futures::channel::mpsc::{Sender, channel};
 use futures::future::{join, join_all};
 use futures::lock::Mutex;
-use futures::{AsyncRead, AsyncWrite, AsyncWriteExt, SinkExt, StreamExt};
+use futures::{SinkExt, StreamExt, stream};
-use hashbrown::{HashMap, HashSet};
+use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api_traits::{Decode, Encode, Request};
+use orchid_api_traits::Request;
+use orchid_base::builtin::ExtInit;
+use orchid_base::clone;
 use orchid_base::format::{FmtCtxImpl, Format};
-use orchid_base::interner::{IStr, IStrv, es, ev, is, iv};
+use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
-use orchid_base::logging::log;
+use orchid_base::logging::Logger;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::{
+use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
-	Client, ClientExt, CommCtx, MsgReaderExt, ReqHandleExt, ReqReaderExt, io_comm,
-};
-use orchid_base::stash::{stash, with_stash};
 use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::atom::AtomHand;
-use crate::ctx::{Ctx, JoinHandle};
+use crate::ctx::Ctx;
 use crate::dealias::{ChildError, ChildErrorKind, walk};
 use crate::expr::{Expr, PathSetBuilder};
 use crate::system::SystemCtor;
 use crate::tree::MemberKind;
 
-pub struct ExtPort {
-	pub input: Pin<Box<dyn AsyncWrite>>,
-	pub output: Pin<Box<dyn AsyncRead>>,
-}
-
 pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 
 /// Data held about an Extension. This is refcounted within [Extension]. It's
@@ -48,128 +42,120 @@ pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 pub struct ExtensionData {
 	name: String,
 	ctx: Ctx,
-	comm_cx: Option<CommCtx>,
+	reqnot: ReqNot<api::HostMsgSet>,
-	join_ext: Option<Box<dyn JoinHandle>>,
-	client: Rc<dyn Client>,
 	systems: Vec<SystemCtor>,
+	logger: Logger,
 	next_pars: RefCell<NonZeroU64>,
+	exiting_snd: Sender<()>,
 	lex_recur: Mutex<HashMap<api::ParsId, Sender<ReqPair<api::SubLex>>>>,
-	strings: RefCell<HashSet<IStr>>,
-	string_vecs: RefCell<HashSet<IStrv>>,
 }
 impl Drop for ExtensionData {
 	fn drop(&mut self) {
-		let client = self.client.clone();
+		let reqnot = self.reqnot.clone();
-		let join_ext = self.join_ext.take().expect("Only called once in Drop");
+		let mut exiting_snd = self.exiting_snd.clone();
-		let comm_cx = self.comm_cx.take().expect("Only used here");
+		(self.ctx.spawn)(Box::pin(async move {
-		stash(async move {
+			reqnot.notify(api::HostExtNotif::Exit).await;
-			client.notify(api::HostExtNotif::Exit).await.unwrap();
+
-			comm_cx.exit().await.unwrap();
+			exiting_snd.send(()).await.unwrap()
-			join_ext.join().await;
+		}))
-		})
 	}
 }
 
 #[derive(Clone)]
 pub struct Extension(Rc<ExtensionData>);
 impl Extension {
-	pub async fn new(mut init: ExtPort, ctx: Ctx) -> io::Result<Self> {
+	pub fn new(init: ExtInit, logger: Logger, msg_logger: Logger, ctx: Ctx) -> io::Result<Self> {
-		api::HostHeader { logger: ctx.logger.to_api() }.encode(init.input.as_mut()).await.unwrap();
-		init.input.flush().await.unwrap();
-
-		let header = api::ExtensionHeader::decode(init.output.as_mut()).await.unwrap();
-		let header2 = header.clone();
 		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| {
-			// context not needed because exit is extension-initiated
+			let init = Rc::new(init);
-			let (client, comm_cx, comm) = io_comm(init.input, init.output);
+			let (exiting_snd, exiting_rcv) = channel::<()>(0);
-			let weak2 = weak;
+			(ctx.spawn)({
-			let weak = weak.clone();
+				clone!(init, weak, ctx);
-			let ctx2 = ctx.clone();
+				Box::pin(async move {
-			let join_ext = ctx.clone().spawn(async move {
+					let rcv_stream = stream(async |mut cx| {
-				comm
+						loop {
-					.listen(
+							cx.emit(init.recv().await).await
-						async |reader| {
+						}
-							with_stash(async {
+					});
+					let mut event_stream = pin!(stream::select(exiting_rcv.map(|()| None), rcv_stream));
+					while let Some(Some(msg)) = event_stream.next().await {
+						if let Some(reqnot) = weak.upgrade().map(|rc| rc.reqnot.clone()) {
+							let reqnot = reqnot.clone();
+							(ctx.spawn)(Box::pin(async move {
+								reqnot.receive(&msg).await;
+							}))
+						}
+					}
+				})
+			});
+			ExtensionData {
+				name: init.name.clone(),
+				exiting_snd,
+				ctx: ctx.clone(),
+				systems: (init.systems.iter().cloned())
+					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
+					.collect(),
+				logger: logger.clone(),
+				next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
+				lex_recur: Mutex::default(),
+				reqnot: ReqNot::new(
+					msg_logger,
+					move |sfn, _| clone!(init; Box::pin(async move { init.send(sfn).await })),
+					clone!(weak; move |notif, _| {
+						clone!(weak; Box::pin(async move {
 						let this = Extension(weak.upgrade().unwrap());
-								let notif = reader.read::<api::ExtHostNotif>().await.unwrap();
-								// logging is never logged because its value will be logged anyway
 						if !matches!(notif, api::ExtHostNotif::Log(_)) {
-									writeln!(log("msg"), "Host received notif {notif:?}").await;
+							writeln!(this.reqnot().logger(), "Host received notif {notif:?}");
 						}
 						match notif {
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
 								let target = this.0.ctx.exprs.get_expr(acq.1).expect("Invalid ticket");
 								this.0.ctx.exprs.give_expr(target)
-									},
+							}
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
+								if this.is_own_sys(rel.0).await {
 									this.0.ctx.exprs.take_expr(rel.1);
-									},
+								} else {
-									api::ExtHostNotif::Log(api::Log { category, message }) =>
+									writeln!(this.reqnot().logger(), "Not our system {:?}", rel.0)
-										write!(log(&es(category).await), "{message}").await,
-									api::ExtHostNotif::Sweeped(data) => {
-										for i in join_all(data.strings.into_iter().map(es)).await {
-											this.0.strings.borrow_mut().remove(&i);
-										}
-										for i in join_all(data.vecs.into_iter().map(ev)).await {
-											this.0.string_vecs.borrow_mut().remove(&i);
 								}
 							},
+							api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str),
 						}
-								Ok(())
+					}))}),
-							})
-							.await
-						},
-						async |mut reader| {
-							with_stash(async {
-								let req = reader.read_req::<api::ExtHostReq>().await.unwrap();
-								let handle = reader.finish().await;
-								// Atom printing and interning is never reported because it generates too much
-								// noise
-								if !matches!(req, api::ExtHostReq::ExtAtomPrint(_))
-									&& !matches!(req, api::ExtHostReq::IntReq(_))
 					{
-									writeln!(log("msg"), "Host received request {req:?}").await;
+						clone!(weak, ctx);
-								}
+						move |hand, req| {
+							clone!(weak, ctx);
+							Box::pin(async move {
 								let this = Self(weak.upgrade().unwrap());
+								if !matches!(req, api::ExtHostReq::ExtAtomPrint(_)) {
+									writeln!(this.reqnot().logger(), "Host received request {req:?}");
+								}
+								let i = this.ctx().i.clone();
 								match req {
-									api::ExtHostReq::Ping(ping) => handle.reply(&ping, &()).await,
+									api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
 									api::ExtHostReq::IntReq(intreq) => match intreq {
-										api::IntReq::InternStr(s) => {
+										api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await,
-											let i = is(&s.0).await;
-											this.0.strings.borrow_mut().insert(i.clone());
-											handle.reply(&s, &i.to_api()).await
-										},
 										api::IntReq::InternStrv(v) => {
-											let tokens = join_all(v.0.iter().map(|m| es(*m))).await;
+											let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await;
-											this.0.strings.borrow_mut().extend(tokens.iter().cloned());
+											hand.handle(&v, &i.i(&tokens).await.to_api()).await
-											let i = iv(&tokens).await;
-											this.0.string_vecs.borrow_mut().insert(i.clone());
-											handle.reply(&v, &i.to_api()).await
-										},
-										api::IntReq::ExternStr(si) => {
-											let i = es(si.0).await;
-											this.0.strings.borrow_mut().insert(i.clone());
-											handle.reply(&si, &i.to_string()).await
 										},
+										api::IntReq::ExternStr(si) =>
+											hand.handle(&si, &Tok::<String>::from_api(si.0, &i).await.rc()).await,
 										api::IntReq::ExternStrv(vi) => {
-											let i = ev(vi.0).await;
+											let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec();
-											this.0.strings.borrow_mut().extend(i.iter().cloned());
+											hand.handle(&vi, &markerv).await
-											this.0.string_vecs.borrow_mut().insert(i.clone());
-											let markerv = i.iter().map(|t| t.to_api()).collect_vec();
-											handle.reply(&vi, &markerv).await
 										},
 									},
 									api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
 										let sys =
 											ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
-										let client = sys.client();
 										let reply =
-											client.request(api::Fwded(fw.0.clone(), *key, body.clone())).await.unwrap();
+											sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
-										handle.reply(fw, &reply).await
+										hand.handle(fw, &reply).await
 									},
 									api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
 										let sys = ctx.system_inst(id).await.unwrap();
-										handle.reply(fw, &sys.request(body.clone()).await).await
+										hand.handle(fw, &sys.request(body.clone()).await).await
 									},
 									api::ExtHostReq::SubLex(sl) => {
 										let (rep_in, mut rep_out) = channel(0);
@@ -179,13 +165,13 @@ impl Extension {
 												lex_g.get(&sl.id).cloned().expect("Sublex for nonexistent lexid");
 											req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
 										}
-										handle.reply(&sl, &rep_out.next().await.unwrap()).await
+										hand.handle(&sl, &rep_out.next().await.unwrap()).await
 									},
 									api::ExtHostReq::ExprReq(expr_req) => match expr_req {
 										api::ExprReq::Inspect(ins @ api::Inspect { target }) => {
 											let expr = ctx.exprs.get_expr(target).expect("Invalid ticket");
-											handle
+											hand
-												.reply(&ins, &api::Inspected {
+												.handle(&ins, &api::Inspected {
 													refcount: expr.strong_count() as u32,
 													location: expr.pos().to_api(),
 													kind: expr.to_api().await,
@@ -196,12 +182,12 @@ impl Extension {
 											let expr = Expr::from_api(expr, PathSetBuilder::new(), ctx.clone()).await;
 											let expr_id = expr.id();
 											ctx.exprs.give_expr(expr);
-											handle.reply(cre, &expr_id).await
+											hand.handle(cre, &expr_id).await
 										},
 									},
 									api::ExtHostReq::LsModule(ref ls @ api::LsModule(_sys, path)) => {
 										let reply: <api::LsModule as Request>::Response = 'reply: {
-											let path = ev(path).await;
+											let path = i.ex(path).await;
 											let root = (ctx.root.read().await.upgrade())
 												.expect("LSModule called when root isn't in context");
 											let root_data = &*root.0.read().await;
@@ -211,13 +197,12 @@ impl Extension {
 													.await
 												{
 													Ok(module) => module,
-													Err(ChildError { kind, .. }) => {
+													Err(ChildError { kind, .. }) =>
 														break 'reply Err(match kind {
 															ChildErrorKind::Private => panic!("Access checking was disabled"),
 															ChildErrorKind::Constant => api::LsModuleError::IsConstant,
 															ChildErrorKind::Missing => api::LsModuleError::InvalidPath,
-														});
+														}),
-													},
 												};
 											let mut members = std::collections::HashMap::new();
 											for (k, v) in &module.members {
@@ -229,7 +214,7 @@ impl Extension {
 											}
 											Ok(api::ModuleInfo { members })
 										};
-										handle.reply(ls, &reply).await
+										hand.handle(ls, &reply).await
 									},
 									api::ExtHostReq::ResolveNames(ref rn) => {
 										let api::ResolveNames { constid, names, sys } = rn;
@@ -241,61 +226,47 @@ impl Extension {
 										};
 										let responses = stream(async |mut cx| {
 											for name in names {
-												cx.emit(match resolver(&ev(*name).await[..]).await {
+												cx.emit(match resolver(&ctx.i.ex(*name).await[..]).await {
-													Ok(abs) => {
+													Ok(abs) => Ok(abs.to_sym(&ctx.i).await.to_api()),
-														let sym = abs.to_sym().await;
+													Err(e) => Err(e.to_api()),
-														this.0.string_vecs.borrow_mut().insert(sym.tok());
-														Ok(sym.to_api())
-													},
-													Err(e) => {
-														(this.0.strings.borrow_mut())
-															.extend(e.iter().map(|e| e.description.clone()));
-														Err(e.to_api())
-													},
 												})
 												.await
 											}
 										})
 										.collect()
 										.await;
-										handle.reply(rn, &responses).await
+										hand.handle(rn, &responses).await
 									},
 									api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
 										let atom = AtomHand::from_api(atom, Pos::None, &mut ctx.clone()).await;
-										let unit = atom.print(&FmtCtxImpl::default()).await;
+										let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
-										handle.reply(eap, &unit.to_api()).await
+										hand.handle(eap, &unit.to_api()).await
 									},
 								}
 							})
-							.await
+						}
 					},
-					)
+				),
-					.await
-					.unwrap();
-			});
-			ExtensionData {
-				name: header2.name.clone(),
-				ctx: ctx2,
-				comm_cx: Some(comm_cx),
-				systems: (header.systems.iter().cloned())
-					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak2.clone()) })
-					.collect(),
-				join_ext: Some(join_ext),
-				next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
-				lex_recur: Mutex::default(),
-				client: Rc::new(client),
-				strings: RefCell::default(),
-				string_vecs: RefCell::default(),
 			}
 		})))
 	}
 	pub fn name(&self) -> &String { &self.0.name }
 	#[must_use]
-	pub fn client(&self) -> &dyn Client { &*self.0.client }
+	pub fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.reqnot }
 	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	#[must_use]
+	pub fn logger(&self) -> &Logger { &self.0.logger }
 	pub fn system_ctors(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
 	#[must_use]
+	pub async fn is_own_sys(&self, id: api::SysId) -> bool {
+		let Some(sys) = self.ctx().system_inst(id).await else {
+			writeln!(self.logger(), "Invalid system ID {id:?}");
+			return false;
+		};
+		Rc::ptr_eq(&self.0, &sys.ext().0)
+	}
+	#[must_use]
 	pub fn next_pars(&self) -> NonZeroU64 {
 		let mut next_pars = self.0.next_pars.borrow_mut();
 		*next_pars = next_pars.checked_add(1).unwrap_or(NonZeroU64::new(1).unwrap());
@@ -303,7 +274,7 @@ impl Extension {
 	}
 	pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: IStr,
+		source: Tok<String>,
 		src: Sym,
 		pos: u32,
 		sys: api::SysId,
@@ -316,10 +287,9 @@ impl Extension {
 		self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
 		let (ret, ()) = join(
 			async {
-				let res = (self.client())
+				let res = (self.reqnot())
 					.request(api::LexExpr { id, pos, sys, src: src.to_api(), text: source.to_api() })
-					.await
+					.await;
-					.unwrap();
 				// collect sender to unblock recursion handler branch before returning
 				self.0.lex_recur.lock().await.remove(&id);
 				res
@@ -336,10 +306,10 @@ impl Extension {
 	}
 	pub fn system_drop(&self, id: api::SysId) {
 		let rc = self.clone();
-		let _ = self.ctx().spawn(with_stash(async move {
+		(self.ctx().spawn)(Box::pin(async move {
-			rc.client().request(api::SystemDrop(id)).await.unwrap();
+			rc.reqnot().request(api::SystemDrop(id)).await;
 			rc.ctx().systems.write().await.remove(&id);
-		}));
+		}))
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }

orchid-host/src/inline.rs

@@ -1,45 +0,0 @@
-#[cfg(feature = "orchid-extension")]
-use orchid_extension as ox;
-
-#[cfg(feature = "orchid-extension")]
-use crate::ctx::Ctx;
-#[cfg(feature = "orchid-extension")]
-use crate::extension::ExtPort;
-
-#[cfg(feature = "orchid-extension")]
-pub async fn ext_inline(builder: ox::entrypoint::ExtensionBuilder, ctx: Ctx) -> ExtPort {
-	use std::io;
-	use std::rc::Rc;
-
-	use futures::io::BufReader;
-	use futures::{AsyncBufReadExt, StreamExt};
-	use orchid_base::logging::log;
-	use unsync_pipe::pipe;
-
-	let (in_stdin, out_stdin) = pipe(1024);
-	let (in_stdout, out_stdout) = pipe(1024);
-	let (in_stderr, out_stderr) = pipe(1024);
-
-	let name = builder.name;
-
-	std::mem::drop(ctx.spawn(async move {
-		let mut lines = BufReader::new(out_stderr).lines();
-		while let Some(line) = lines.next().await {
-			match line {
-				Ok(line) => writeln!(log("stderr"), "inline {name} err> {line}").await,
-				Err(e) => match e.kind() {
-					io::ErrorKind::BrokenPipe | io::ErrorKind::UnexpectedEof => break,
-					_ => panic!("Error while reading stderr {e}"),
-				},
-			}
-		}
-	}));
-
-	builder.build(ox::ext_port::ExtPort {
-		input: Box::pin(out_stdin),
-		output: Box::pin(in_stdout),
-		log: Box::pin(in_stderr),
-		spawn: Rc::new(move |fut| std::mem::drop(ctx.spawn(fut))),
-	});
-	ExtPort { input: Box::pin(in_stdin), output: Box::pin(out_stdout) }
-}

orchid-host/src/lex.rs

@@ -1,11 +1,10 @@
-use std::collections::VecDeque;
+use std::rc::Rc;
-use std::ops::Range;
 
 use futures::FutureExt;
 use futures::lock::Mutex;
 use orchid_base::clone;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv, report};
+use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
-use orchid_base::interner::{IStr, is};
+use orchid_base::interner::Tok;
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
@@ -21,16 +20,15 @@ use crate::system::System;
 
 pub struct LexCtx<'a> {
 	pub systems: &'a [System],
-	pub source: &'a IStr,
+	pub source: &'a Tok<String>,
 	pub path: &'a Sym,
 	pub tail: &'a str,
 	pub sub_trees: &'a mut Vec<Expr>,
 	pub ctx: &'a Ctx,
-	pub produced: &'a mut VecDeque<ParsTokTree>,
 }
 impl<'a> LexCtx<'a> {
 	#[must_use]
-	pub fn sub<'b>(&'b mut self, pos: u32, produced: &'b mut VecDeque<ParsTokTree>) -> LexCtx<'b>
+	pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
 	where 'a: 'b {
 		LexCtx {
 			source: self.source,
@@ -39,7 +37,6 @@ impl<'a> LexCtx<'a> {
 			systems: self.systems,
 			sub_trees: &mut *self.sub_trees,
 			ctx: self.ctx,
-			produced,
 		}
 	}
 	#[must_use]
@@ -49,7 +46,6 @@ impl<'a> LexCtx<'a> {
 	pub fn set_pos(&mut self, pos: u32) { self.tail = &self.source[pos as usize..] }
 	pub fn push_pos(&mut self, delta: u32) { self.set_pos(self.get_pos() + delta) }
 	pub fn set_tail(&mut self, tail: &'a str) { self.tail = tail }
-	pub fn pos_from(&self, tail: &'a str) -> u32 { (self.source.len() - tail.len()) as u32 }
 	#[must_use]
 	pub fn strip_prefix(&mut self, tgt: &str) -> bool {
 		if let Some(src) = self.tail.strip_prefix(tgt) {
@@ -64,7 +60,7 @@ impl<'a> LexCtx<'a> {
 	}
 	#[must_use]
 	pub async fn des_subtree(&mut self, tree: &api::TokenTree, exprs: ExprStore) -> ParsTokTree {
-		ParsTokTree::from_api(tree, &mut { exprs }, &mut self.ctx.clone(), self.path).await
+		ParsTokTree::from_api(tree, &mut { exprs }, &mut self.ctx.clone(), self.path, &self.ctx.i).await
 	}
 	#[must_use]
 	pub fn strip_char(&mut self, tgt: char) -> bool {
@@ -85,138 +81,60 @@ impl<'a> LexCtx<'a> {
 		self.tail = rest;
 		matches
 	}
-	pub fn pop_char(&mut self) -> Option<char> {
-		let mut chars = self.tail.chars();
-		let ret = chars.next()?;
-		self.tail = chars.as_str();
-		Some(ret)
-	}
-	pub fn sr_to(&self, start: u32) -> SrcRange { self.sr(start..self.get_pos()) }
-	pub fn sr(&self, range: Range<u32>) -> SrcRange { SrcRange::new(range, self.path) }
 }
 
-pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<bool> {
+pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
-	ctx.trim(unrep_space);
-	if ctx.tail.is_empty() {
-		return Ok(false);
-	}
 	let start = ctx.get_pos();
+	assert!(
+		!ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space),
+		"Lexing empty string or whitespace to token!\n\
+    Invocations of lex_tok should check for empty string"
+	);
 	let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
 		ParsTok::BR
 	} else if let Some(tail) = (ctx.tail.starts_with(name_start).then_some(ctx.tail))
 		.and_then(|t| t.trim_start_matches(name_char).strip_prefix("::"))
 	{
 		let name = &ctx.tail[..ctx.tail.len() - tail.len() - "::".len()];
-		let mut produced = VecDeque::new();
+		ctx.set_tail(tail);
-		let mut sub_cx = ctx.sub(ctx.pos_from(tail), &mut produced);
+		let body = lex_once(ctx).boxed_local().await?;
-		if !lex_once(&mut sub_cx).boxed_local().await? {
+		ParsTok::NS(ctx.ctx.i.i(name).await, Box::new(body))
-			return Err(mk_errv(
-				is("Unexpected end of source text").await,
-				":: cannot be the last token",
-				[SrcRange::new(start..ctx.get_pos(), ctx.path)],
-			));
-		}
-		let pos = sub_cx.get_pos();
-		ctx.set_pos(pos);
-		let body = produced.pop_front().expect("lex_once returned true");
-		ctx.produced.extend(produced.into_iter());
-		ParsTok::NS(is(name).await, Box::new(body))
 	} else if ctx.strip_prefix("--[") {
 		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
 			return Err(mk_errv(
-				is("Unterminated block comment").await,
+				ctx.ctx.i.i("Unterminated block comment").await,
 				"This block comment has no ending ]--",
 				[SrcRange::new(start..start + 3, ctx.path)],
 			));
 		};
 		ctx.set_tail(tail);
-		ParsTok::Comment(is(cmt).await)
+		ParsTok::Comment(Rc::new(cmt.to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
-		ParsTok::Comment(is(&tail[2..end]).await)
+		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
 		// fanciness like \$placeh in templates is resolved in the macro engine.
-		let start = ctx.get_pos();
+		ctx.set_tail(tail);
-		let mut produced = VecDeque::new();
+		let arg = lex_once(ctx).boxed_local().await?;
-		let mut sub_cx = ctx.sub(ctx.pos_from(tail), &mut produced);
-		if !lex_once(&mut sub_cx).boxed_local().await? {
-			return Err(mk_errv(
-				is("Unexpected end of file").await,
-				"Expected a lambda argument and body",
-				[SrcRange::new(start..ctx.get_pos(), ctx.path)],
-			));
-		}
-		let pos = sub_cx.get_pos();
-		ctx.set_pos(pos);
-		let arg = produced.pop_front().expect("lex_once returned true");
-		ctx.produced.extend(produced);
 		ctx.trim_ws();
 		ParsTok::LambdaHead(Box::new(arg))
 	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
-		let mut body = VecDeque::new();
+		let mut body = Vec::new();
 		ctx.trim_ws();
 		while !ctx.strip_char(*rp) {
-			let mut sub_cx = ctx.sub(ctx.get_pos(), &mut body);
+			if ctx.tail.is_empty() {
-			if !lex_once(&mut sub_cx).boxed_local().await? {
 				return Err(mk_errv(
-					is("unclosed paren").await,
+					ctx.ctx.i.i("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
 					[SrcRange::new(start..start + 1, ctx.path)],
 				));
 			}
-			let pos = sub_cx.get_pos();
+			body.push(lex_once(ctx).boxed_local().await?);
-			ctx.set_pos(pos);
 			ctx.trim_ws();
 		}
-		ParsTok::S(*paren, body.into_iter().collect())
+		ParsTok::S(*paren, body)
-	} else if let Some(res) = sys_lex(ctx).await {
-		let token = res?;
-		ctx.produced.extend(token);
-		return Ok(true);
-	} else if ctx.tail.starts_with(name_start) {
-		ParsTok::Name(is(ctx.get_start_matches(name_char)).await)
-	} else if ctx.tail.starts_with(op_char) {
-		let whole_tail = ctx.tail;
-		ctx.pop_char().expect("The above check would have failed");
-		let mut tail_after_op = ctx.tail;
-
-		let mut lookahead = Vec::new();
-		while !ctx.tail.is_empty() && ctx.tail.starts_with(op_char) {
-			match sys_lex(ctx).await {
-				None => {
-					ctx.pop_char();
-					tail_after_op = ctx.tail;
-				},
-				Some(sys_res) => {
-					match sys_res {
-						Err(e) => report(e),
-						Ok(tokv) => lookahead = tokv,
-					}
-					break;
-				},
-			}
-		}
-
-		let op_str = &whole_tail[0..whole_tail.len() - tail_after_op.len()];
-		ctx.produced.push_back(ParsTok::Name(is(op_str).await).at(ctx.sr_to(start)));
-		ctx.produced.extend(lookahead);
-		return Ok(true);
 	} else {
-		return Err(mk_errv(
-			is("Unrecognized character").await,
-			"The following syntax is meaningless.",
-			[SrcRange::new(start..start + 1, ctx.path)],
-		));
-	};
-	ctx.produced.push_back(ParsTokTree { tok, sr: ctx.sr_to(start) });
-	Ok(true)
-}
-
-/// Parse one token via any of the systems, if we can
-///
-/// This function never writes lookahead
-pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
 		for sys in ctx.systems {
 			let mut errors = Vec::new();
 			if ctx.tail.starts_with(|c| sys.can_lex(c)) {
@@ -230,27 +148,11 @@ pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
 						clone!(temp_store_cb);
 						async move {
 							let mut ctx_g = ctx_lck.lock().await;
-						let mut produced = VecDeque::new();
+							match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
-						let mut sub_cx = ctx_g.sub(pos, &mut produced);
+								Ok(t) => Some(api::SubLexed {
-						let lex_res = lex_once(&mut sub_cx).boxed_local().await;
+									pos: t.sr.end(),
-						let pos1 = sub_cx.get_pos();
+									tree: ctx_g.ser_subtree(t, temp_store_cb.clone()).await,
-						ctx_g.set_pos(pos1);
+								}),
-						match lex_res {
-							Ok(false) => {
-								errors_lck.lock().await.push(mk_errv(
-									is("End of file").await,
-									"Unexpected end of source text",
-									[ctx_g.sr_to(pos)],
-								));
-								None
-							},
-							Ok(true) => {
-								let tok = produced.pop_front().unwrap();
-								Some(api::SubLexed {
-									pos: tok.sr.end(),
-									tree: ctx_g.ser_subtree(tok, temp_store_cb.clone()).await,
-								})
-							},
 								Err(e) => {
 									errors_lck.lock().await.push(e);
 									None
@@ -261,45 +163,56 @@ pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
 					.await;
 				match lx {
 					Err(e) =>
-					return Some(Err(errors.into_iter().fold(OrcErrv::from_api(&e).await, |a, b| a + b))),
+						return Err(
+							errors.into_iter().fold(OrcErrv::from_api(&e, &ctx.ctx.i).await, |a, b| a + b),
+						),
 					Ok(Some(lexed)) => {
 						ctx.set_pos(lexed.pos);
-					let mut stable_trees = Vec::new();
+						let lexed_tree = ctx.des_subtree(&lexed.expr, temp_store).await;
-					for tok in lexed.expr {
+						let stable_tree = recur(lexed_tree, &|tt, r| {
-						stable_trees.push(recur(ctx.des_subtree(&tok, temp_store.clone()).await, &|tt, r| {
 							if let ParsTok::NewExpr(expr) = tt.tok {
 								return ParsTok::Handle(expr).at(tt.sr);
 							}
 							r(tt)
-						}));
+						});
-					}
+						return Ok(stable_tree);
-					return Some(Ok(stable_trees));
 					},
 					Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
-					Some(errors) => return Some(Err(errors)),
+						Some(errors) => return Err(errors),
 						None => continue,
 					},
 				}
 			}
 		}
-	None
+		if ctx.tail.starts_with(name_start) {
+			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(name_char)).await)
+		} else if ctx.tail.starts_with(op_char) {
+			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(op_char)).await)
+		} else {
+			return Err(mk_errv(
+				ctx.ctx.i.i("Unrecognized character").await,
+				"The following syntax is meaningless.",
+				[SrcRange::new(start..start + 1, ctx.path)],
+			));
+		}
+	};
+	Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
 }
 
-pub async fn lex(text: IStr, path: Sym, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
+pub async fn lex(
+	text: Tok<String>,
+	path: Sym,
+	systems: &[System],
+	ctx: &Ctx,
+) -> OrcRes<Vec<ParsTokTree>> {
 	let mut sub_trees = Vec::new();
-	let mut produced = VecDeque::new();
+	let mut ctx =
-	let mut ctx = LexCtx {
+		LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };
-		source: &text,
+	let mut tokv = Vec::new();
-		sub_trees: &mut sub_trees,
-		tail: &text[..],
-		systems,
-		path: &path,
-		ctx,
-		produced: &mut produced,
-	};
 	ctx.trim(unrep_space);
-	while lex_once(&mut ctx).await? {
+	while !ctx.tail.is_empty() {
+		tokv.push(lex_once(&mut ctx).await?);
 		ctx.trim(unrep_space);
 	}
-	Ok(produced.into())
+	Ok(tokv)
 }

orchid-host/src/lib.rs

@@ -3,18 +3,13 @@ use orchid_api as api;
 pub mod atom;
 pub mod ctx;
 pub mod dealias;
-#[cfg(feature = "tokio")]
-pub mod dylib;
 pub mod execute;
 pub mod expr;
 pub mod expr_store;
 pub mod extension;
-pub mod inline;
 pub mod lex;
-pub mod logger;
 pub mod parse;
 pub mod parsed;
-#[cfg(feature = "tokio")]
 pub mod subprocess;
 mod sys_parser;
 pub mod system;

orchid-host/src/logger.rs

@@ -1,84 +0,0 @@
-use std::fmt::Arguments;
-use std::fs::File;
-use std::io::{Write, stderr};
-use std::rc::Rc;
-
-use futures::future::LocalBoxFuture;
-use hashbrown::HashMap;
-use itertools::Itertools;
-use orchid_base::logging::{LogWriter, Logger};
-
-use crate::api;
-
-pub struct LogWriterImpl(api::LogStrategy);
-impl LogWriter for LogWriterImpl {
-	fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()> {
-		Box::pin(async move {
-			match &self.0 {
-				api::LogStrategy::Discard => (),
-				api::LogStrategy::Default => {
-					stderr().write_fmt(fmt).expect("Could not write to stderr!");
-					stderr().flush().expect("Could not flush stderr")
-				},
-				api::LogStrategy::File { path, .. } => {
-					let mut file = (File::options().write(true).create(true).truncate(false).open(path))
-						.unwrap_or_else(|e| panic!("Could not open {path}: {e}"));
-					file.write_fmt(fmt).unwrap_or_else(|e| panic!("Could not write to {path}: {e}"));
-				},
-			}
-		})
-	}
-}
-
-#[derive(Clone, Default)]
-pub struct LoggerImpl {
-	routing: HashMap<String, api::LogStrategy>,
-	default: Option<api::LogStrategy>,
-}
-impl LoggerImpl {
-	pub fn to_api(&self) -> api::Logger {
-		api::Logger {
-			default: self.default.clone(),
-			routing: self.routing.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
-		}
-	}
-
-	pub fn new(
-		default: Option<api::LogStrategy>,
-		strats: impl IntoIterator<Item = (String, api::LogStrategy)>,
-	) -> Self {
-		Self { routing: strats.into_iter().collect(), default }
-	}
-	pub fn set_default(&mut self, strat: api::LogStrategy) { self.default = Some(strat) }
-	pub fn clear_default(&mut self) { self.default = None }
-	pub fn set_category(&mut self, category: &str, strat: api::LogStrategy) {
-		self.routing.insert(category.to_string(), strat);
-	}
-	pub fn with_default(mut self, strat: api::LogStrategy) -> Self {
-		self.set_default(strat);
-		self
-	}
-	pub fn with_category(mut self, category: &str, strat: api::LogStrategy) -> Self {
-		self.set_category(category, strat);
-		self
-	}
-	pub async fn log(&self, category: &str, msg: impl AsRef<str>) {
-		writeln!(self.writer(category), "{}", msg.as_ref()).await
-	}
-	pub fn has_category(&self, category: &str) -> bool { self.routing.contains_key(category) }
-	pub async fn log_buf(&self, category: &str, event: impl AsRef<str>, buf: &[u8]) {
-		if std::env::var("ORCHID_LOG_BUFFERS").is_ok_and(|v| !v.is_empty()) {
-			let data = buf.iter().map(|b| format!("{b:02x}")).join(" ");
-			writeln!(self.writer(category), "{}: [{data}]", event.as_ref()).await
-		}
-	}
-}
-impl Logger for LoggerImpl {
-	fn writer(&self, category: &str) -> Rc<dyn LogWriter> {
-		Rc::new(LogWriterImpl(self.strat(category).clone()))
-	}
-	fn strat(&self, category: &str) -> api::LogStrategy {
-		(self.routing.get(category).cloned().or(self.default.clone()))
-			.expect("Invalid category and catchall logger not set")
-	}
-}

orchid-host/src/parse.rs

@@ -1,11 +1,12 @@
 use futures::FutureExt;
 use itertools::Itertools;
-use orchid_base::error::{OrcRes, mk_errv, report};
+use orchid_base::error::{OrcRes, Reporter, mk_errv};
 use orchid_base::format::fmt;
-use orchid_base::interner::{IStr, is};
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::name::Sym;
 use orchid_base::parse::{
-	Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, try_pop_no_fluff,
+	Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
+	try_pop_no_fluff,
 };
 use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
@@ -21,6 +22,12 @@ pub struct HostParseCtxImpl<'a> {
 	pub ctx: Ctx,
 	pub src: Sym,
 	pub systems: &'a [System],
+	pub rep: &'a Reporter,
+}
+
+impl ParseCtx for HostParseCtxImpl<'_> {
+	fn rep(&self) -> &Reporter { self.rep }
+	fn i(&self) -> &Interner { &self.ctx.i }
 }
 
 impl HostParseCtx for HostParseCtxImpl<'_> {
@@ -29,7 +36,7 @@ impl HostParseCtx for HostParseCtxImpl<'_> {
 	fn src_path(&self) -> Sym { self.src.clone() }
 }
 
-pub trait HostParseCtx {
+pub trait HostParseCtx: ParseCtx {
 	#[must_use]
 	fn ctx(&self) -> &Ctx;
 	#[must_use]
@@ -40,14 +47,14 @@ pub trait HostParseCtx {
 
 pub async fn parse_items(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, IStr>,
+	path: Substack<'_, Tok<String>>,
 	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let lines = line_items(items).await;
+	let lines = line_items(ctx, items).await;
 	let mut line_ok = Vec::new();
 	for Parsed { output: comments, tail } in lines {
 		match parse_item(ctx, path.clone(), comments, tail).boxed_local().await {
-			Err(e) => report(e),
+			Err(e) => ctx.rep().report(e),
 			Ok(l) => line_ok.extend(l),
 		}
 	}
@@ -56,23 +63,23 @@ pub async fn parse_items(
 
 pub async fn parse_item(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, IStr>,
+	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
 	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	match item.pop_front() {
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
-			n if *n == is("export").await => match try_pop_no_fluff(postdisc).await? {
+			n if *n == ctx.i().i("export").await => match try_pop_no_fluff(ctx, postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
 					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
 				Parsed { output, tail: _ } => Err(mk_errv(
-					is("Malformed export").await,
+					ctx.i().i("Malformed export").await,
 					"`export` can either prefix other lines or list names inside ( )",
 					[output.sr()],
 				)),
 			},
-			n if *n == is("import").await => {
+			n if *n == ctx.i().i("import").await => {
-				let imports = parse_import(postdisc).await?;
+				let imports = parse_import(ctx, postdisc).await?;
 				Ok(Vec::from_iter(imports.into_iter().map(|t| Item {
 					comments: comments.clone(),
 					sr: t.sr.clone(),
@@ -81,29 +88,33 @@ pub async fn parse_item(
 			},
 			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
 		},
-		Some(_) =>
+		Some(_) => Err(mk_errv(
-			Err(mk_errv(is("Expected a line type").await, "All lines must begin with a keyword", [
+			ctx.i().i("Expected a line type").await,
-				item.sr()
+			"All lines must begin with a keyword",
-			])),
+			[item.sr()],
+		)),
 		None => unreachable!("These lines are filtered and aggregated in earlier stages"),
 	}
 }
 
-pub async fn parse_import<'a>(tail: ParsSnippet<'a>) -> OrcRes<Vec<Import>> {
+pub async fn parse_import<'a>(
-	let Parsed { output: imports, tail } = parse_multiname(tail).await?;
+	ctx: &impl HostParseCtx,
-	expect_end(tail).await?;
+	tail: ParsSnippet<'a>,
+) -> OrcRes<Vec<Import>> {
+	let Parsed { output: imports, tail } = parse_multiname(ctx, tail).await?;
+	expect_end(ctx, tail).await?;
 	Ok(imports)
 }
 
 pub async fn parse_exportable_item<'a>(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, IStr>,
+	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
 	exported: bool,
-	discr: IStr,
+	discr: Tok<String>,
 	tail: ParsSnippet<'a>,
 ) -> OrcRes<Vec<Item>> {
-	let kind = if discr == is("mod").await {
+	let kind = if discr == ctx.i().i("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
 		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::Mod(body) })
 	} else if let Some(parser) = ctx.systems().find_map(|s| s.get_parser(discr.clone())) {
@@ -116,7 +127,7 @@ pub async fn parse_exportable_item<'a>(
 	} else {
 		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
 		return Err(mk_errv(
-			is("Unrecognized line type").await,
+			ctx.i().i("Unrecognized line type").await,
 			format!("Line types are: mod, {ext_lines}"),
 			[tail.prev().sr()],
 		));
@@ -126,25 +137,25 @@ pub async fn parse_exportable_item<'a>(
 
 pub async fn parse_module<'a>(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, IStr>,
+	path: Substack<'_, Tok<String>>,
 	tail: ParsSnippet<'a>,
-) -> OrcRes<(IStr, ParsedModule)> {
+) -> OrcRes<(Tok<String>, ParsedModule)> {
-	let (name, tail) = match try_pop_no_fluff(tail).await? {
+	let (name, tail) = match try_pop_no_fluff(ctx, tail).await? {
 		Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
 		Parsed { output, .. } => {
 			return Err(mk_errv(
-				is("Missing module name").await,
+				ctx.i().i("Missing module name").await,
-				format!("A name was expected, {} was found", fmt(output).await),
+				format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
 				[output.sr()],
 			));
 		},
 	};
-	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?;
+	let Parsed { output, tail: surplus } = try_pop_no_fluff(ctx, tail).await?;
-	expect_end(surplus).await?;
+	expect_end(ctx, surplus).await?;
 	let Some(body) = output.as_s(Paren::Round) else {
 		return Err(mk_errv(
-			is("Expected module body").await,
+			ctx.i().i("Expected module body").await,
-			format!("A ( block ) was expected, {} was found", fmt(output).await),
+			format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await),
 			[output.sr()],
 		));
 	};

orchid-host/src/parsed.rs

@@ -6,7 +6,7 @@ use futures::future::{LocalBoxFuture, join_all};
 use hashbrown::HashSet;
 use itertools::Itertools;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::interner::{IStr, IStrv};
+use orchid_base::interner::Tok;
 use orchid_base::location::SrcRange;
 use orchid_base::parse::{Comment, Import};
 use orchid_base::tl_cache;
@@ -57,10 +57,10 @@ impl Format for Item {
 			ItemKind::Member(mem) => match &mem.kind {
 				ParsedMemberKind::Const(_, sys) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
-						.units([mem.name.to_string().into(), sys.print(c).await]),
+						.units([mem.name.rc().into(), sys.print(c).await]),
 				ParsedMemberKind::Mod(module) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
-						.units([mem.name.to_string().into(), module.print(c).boxed_local().await]),
+						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
 			},
 		};
 		tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
@@ -69,14 +69,14 @@ impl Format for Item {
 }
 
 pub struct ParsedMember {
-	pub name: IStr,
+	pub name: Tok<String>,
 	pub exported: bool,
 	pub kind: ParsedMemberKind,
 }
 impl ParsedMember {
 	#[must_use]
-	pub fn name(&self) -> IStr { self.name.clone() }
+	pub fn name(&self) -> Tok<String> { self.name.clone() }
-	pub fn new(exported: bool, name: IStr, kind: impl Into<ParsedMemberKind>) -> Self {
+	pub fn new(exported: bool, name: Tok<String>, kind: impl Into<ParsedMemberKind>) -> Self {
 		Self { exported, name, kind: kind.into() }
 	}
 }
@@ -89,14 +89,17 @@ impl Debug for ParsedMember {
 	}
 }
 
-pub(crate) type ParsedExprCallback = Rc<dyn for<'a> Fn(&'a [IStr]) -> LocalBoxFuture<'a, Expr>>;
+pub(crate) type ParsedExprCallback =
+	Rc<dyn for<'a> Fn(&'a [Tok<String>]) -> LocalBoxFuture<'a, Expr>>;
 
 pub struct ParsedExpr {
 	pub(crate) debug: String,
 	pub(crate) callback: ParsedExprCallback,
 }
 impl ParsedExpr {
-	pub async fn run(self, imported_names: &[IStr]) -> Expr { (self.callback)(imported_names).await }
+	pub async fn run(self, imported_names: &[Tok<String>]) -> Expr {
+		(self.callback)(imported_names).await
+	}
 }
 impl fmt::Debug for ParsedExpr {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.debug) }
@@ -112,7 +115,7 @@ impl From<ParsedModule> for ParsedMemberKind {
 }
 #[derive(Debug, Default)]
 pub struct ParsedModule {
-	pub exports: Vec<IStr>,
+	pub exports: Vec<Tok<String>>,
 	pub items: Vec<Item>,
 	pub use_prelude: bool,
 }
@@ -138,7 +141,7 @@ impl ParsedModule {
 		(self.items.iter())
 			.filter_map(|it| if let ItemKind::Import(i) = &it.kind { Some(i) } else { None })
 	}
-	pub fn default_item(self, name: IStr, sr: SrcRange) -> Item {
+	pub fn default_item(self, name: Tok<String>, sr: SrcRange) -> Item {
 		let mem = ParsedMember { exported: true, name, kind: ParsedMemberKind::Mod(self) };
 		Item { comments: vec![], sr, kind: ItemKind::Member(mem) }
 	}
@@ -147,7 +150,7 @@ impl Tree for ParsedModule {
 	type Ctx<'a> = ();
 	async fn child(
 		&self,
-		key: IStr,
+		key: Tok<String>,
 		public_only: bool,
 		(): &mut Self::Ctx<'_>,
 	) -> ChildResult<'_, Self> {
@@ -165,7 +168,7 @@ impl Tree for ParsedModule {
 		}
 		ChildResult::Err(ChildErrorKind::Missing)
 	}
-	fn children(&self, public_only: bool) -> HashSet<IStr> {
+	fn children(&self, public_only: bool) -> HashSet<Tok<String>> {
 		let mut public: HashSet<_> = self.exports.iter().cloned().collect();
 		if !public_only {
 			public.extend(
@@ -194,11 +197,11 @@ impl Format for ParsedModule {
 /// point to a module and rule_loc selects a macro rule within that module
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct ConstPath {
-	steps: IStrv,
+	steps: Tok<Vec<Tok<String>>>,
 }
 impl ConstPath {
 	#[must_use]
-	pub fn to_const(steps: IStrv) -> Self { Self { steps } }
+	pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps } }
 }
 
 pub async fn tt_to_api(exprs: &mut ExprStore, subtree: ParsTokTree) -> api::TokenTree {

orchid-host/src/subprocess.rs

@@ -1,36 +1,102 @@
-use std::{io, process};
+use std::cell::RefCell;
+use std::io::{self, Write};
+use std::pin::Pin;
 
+use async_process::{self, Child, ChildStdin, ChildStdout};
+use futures::future::LocalBoxFuture;
 use futures::io::BufReader;
-use futures::{self, AsyncBufReadExt, StreamExt};
+use futures::lock::Mutex;
-use orchid_base::logging::log;
+use futures::{self, AsyncBufReadExt, AsyncWriteExt};
-use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
+use orchid_api_traits::{Decode, Encode};
+use orchid_base::builtin::{ExtInit, ExtPort};
+use orchid_base::logging::Logger;
+use orchid_base::msg::{recv_msg, send_msg};
 
+use crate::api;
 use crate::ctx::Ctx;
-use crate::extension::ExtPort;
 
-pub async fn ext_command(cmd: process::Command, ctx: Ctx) -> io::Result<ExtPort> {
+pub async fn ext_command(
-	let name = cmd.get_program().to_string_lossy().to_string();
+	cmd: std::process::Command,
-	let mut child = tokio::process::Command::from(cmd)
+	logger: Logger,
-		.stdin(process::Stdio::piped())
+	msg_logs: Logger,
-		.stdout(process::Stdio::piped())
+	ctx: Ctx,
-		.stderr(process::Stdio::piped())
+) -> io::Result<ExtInit> {
+	let mut child = async_process::Command::from(cmd)
+		.stdin(async_process::Stdio::piped())
+		.stdout(async_process::Stdio::piped())
+		.stderr(async_process::Stdio::piped())
 		.spawn()?;
-	std::thread::spawn(|| {});
+	let mut stdin = child.stdin.take().unwrap();
-	let stdin = child.stdin.take().unwrap();
+	api::HostHeader { log_strategy: logger.strat(), msg_logs: msg_logs.strat() }
-	let stdout = child.stdout.take().unwrap();
+		.encode(Pin::new(&mut stdin))
+		.await;
+	let mut stdout = child.stdout.take().unwrap();
+	let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await;
 	let child_stderr = child.stderr.take().unwrap();
-	let _ = ctx.spawn(Box::pin(async move {
+	(ctx.spawn)(Box::pin(async move {
-		let _ = child;
+		let mut reader = BufReader::new(child_stderr);
-		let mut lines = BufReader::new(child_stderr.compat()).lines();
+		loop {
-		while let Some(line) = lines.next().await {
+			let mut buf = String::new();
-			match line {
+			if 0 == reader.read_line(&mut buf).await.unwrap() {
-				Ok(line) => writeln!(log("stderr"), "subproc {name} err> {line}").await,
+				break;
-				Err(e) => match e.kind() {
-					io::ErrorKind::BrokenPipe | io::ErrorKind::UnexpectedEof => break,
-					_ => panic!("Error while reading stderr {e}"),
-				},
 			}
+			logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
 		}
 	}));
-	Ok(ExtPort { input: Box::pin(stdin.compat_write()), output: Box::pin(stdout.compat()) })
+	Ok(ExtInit {
+		port: Box::new(Subprocess {
+			name: header.name.clone(),
+			child: RefCell::new(Some(child)),
+			stdin: Some(Mutex::new(Box::pin(stdin))),
+			stdout: Mutex::new(Box::pin(stdout)),
+			ctx,
+		}),
+		header,
+	})
+}
+
+pub struct Subprocess {
+	name: String,
+	child: RefCell<Option<Child>>,
+	stdin: Option<Mutex<Pin<Box<ChildStdin>>>>,
+	stdout: Mutex<Pin<Box<ChildStdout>>>,
+	ctx: Ctx,
+}
+impl Drop for Subprocess {
+	fn drop(&mut self) {
+		let mut child = self.child.borrow_mut().take().unwrap();
+		let name = self.name.clone();
+		if std::thread::panicking() {
+			eprintln!("Killing extension {name}");
+			// we don't really care to handle errors here
+			let _: Result<_, _> = std::io::stderr().flush();
+			let _: Result<_, _> = child.kill();
+			return;
+		}
+		let stdin = self.stdin.take().unwrap();
+		(self.ctx.spawn)(Box::pin(async move {
+			stdin.lock().await.close().await.unwrap();
+			let status = (child.status().await)
+				.unwrap_or_else(|e| panic!("{e}, extension {name} exited with error"));
+			assert!(status.success(), "Extension {name} exited with error {status}");
+		}))
+	}
+}
+impl ExtPort for Subprocess {
+	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
+		Box::pin(async {
+			send_msg(Pin::new(&mut *self.stdin.as_ref().unwrap().lock().await), msg).await.unwrap()
+		})
+	}
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
+		Box::pin(async {
+			std::io::Write::flush(&mut std::io::stderr()).unwrap();
+			match recv_msg(self.stdout.lock().await.as_mut()).await {
+				Ok(msg) => Some(msg),
+				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
+				Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => None,
+				Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
+			}
+		})
+	}
 }

orchid-host/src/sys_parser.rs

@@ -2,11 +2,11 @@ use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 use orchid_base::error::{OrcErrv, OrcRes};
-use orchid_base::interner::{IStr, es};
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::parse::Comment;
-use orchid_base::reqnot::ClientExt;
+use orchid_base::reqnot::Requester;
 use orchid_base::tree::ttv_from_api;
 use substack::Substack;
 
@@ -22,7 +22,7 @@ pub struct Parser {
 	pub(crate) system: System,
 	pub(crate) idx: u16,
 }
-type ModPath<'a> = Substack<'a, IStr>;
+type ModPath<'a> = Substack<'a, Tok<String>>;
 
 impl Parser {
 	pub async fn parse(
@@ -39,7 +39,7 @@ impl Parser {
 		let line =
 			join_all((line.into_iter()).map(|t| async { tt_to_api(&mut temp_store.clone(), t).await }))
 				.await;
-		let mod_path = ctx.src_path().suffix(path.unreverse()).await;
+		let mod_path = ctx.src_path().suffix(path.unreverse(), self.system.i()).await;
 		let comments = comments.iter().map(Comment::to_api).collect_vec();
 		let req = api::ParseLine {
 			idx: self.idx,
@@ -50,16 +50,17 @@ impl Parser {
 			comments,
 			line,
 		};
-		match self.system.client().request(req).await.unwrap() {
+		match self.system.reqnot().request(req).await {
 			Ok(parsed_v) =>
 				conv(parsed_v, path, callback, &mut ConvCtx {
+					i: self.system.i(),
 					mod_path: &mod_path,
 					ext_exprs: &mut temp_store,
 					src_path: &src_path,
 					sys: &self.system,
 				})
 				.await,
-			Err(e) => Err(OrcErrv::from_api(&e).await),
+			Err(e) => Err(OrcErrv::from_api(&e, &self.system.ctx().i).await),
 		}
 	}
 }
@@ -68,12 +69,13 @@ struct ConvCtx<'a> {
 	sys: &'a System,
 	mod_path: &'a Sym,
 	src_path: &'a Sym,
+	i: &'a Interner,
 	ext_exprs: &'a mut ExprStore,
 }
 async fn conv(
 	parsed_v: Vec<api::ParsedLine>,
-	module: Substack<'_, IStr>,
+	module: Substack<'_, Tok<String>>,
-	callback: &'_ mut impl AsyncFnMut(Substack<'_, IStr>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	callback: &'_ mut impl AsyncFnMut(Substack<'_, Tok<String>>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
 	ctx: &mut ConvCtx<'_>,
 ) -> OrcRes<Vec<Item>> {
 	let mut items = Vec::new();
@@ -83,12 +85,12 @@ async fn conv(
 				(name, exported, kind),
 			api::ParsedLineKind::Recursive(rec) => {
 				let tokens =
-					ttv_from_api(rec, ctx.ext_exprs, &mut ctx.sys.ctx().clone(), ctx.src_path).await;
+					ttv_from_api(rec, ctx.ext_exprs, &mut ctx.sys.ctx().clone(), ctx.src_path, ctx.i).await;
 				items.extend(callback(module.clone(), tokens).await?);
 				continue;
 			},
 		};
-		let name = es(name).await;
+		let name = ctx.i.ex(name).await;
 		let mem_path = module.push(name.clone());
 		let mkind = match kind {
 			api::ParsedMemberKind::Module { lines, use_prelude } => {
@@ -96,16 +98,16 @@ async fn conv(
 				ParsedMemberKind::Mod(ParsedModule::new(use_prelude, items))
 			},
 			api::ParsedMemberKind::Constant(cid) => {
-				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(mem_path.unreverse()).await);
+				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(mem_path.unreverse(), ctx.i).await);
 				ParsedMemberKind::Const(cid, ctx.sys.clone())
 			},
 		};
 		items.push(Item {
 			comments: join_all(
-				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone())),
+				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone(), ctx.i)),
 			)
 			.await,
-			sr: SrcRange::from_api(&parsed.source_range).await,
+			sr: SrcRange::from_api(&parsed.source_range, ctx.i).await,
 			kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
 		})
 	}

orchid-host/src/system.rs

@@ -12,12 +12,10 @@ use memo_map::MemoMap;
 use orchid_base::char_filter::char_filter_match;
 use orchid_base::error::{OrcRes, mk_errv_floating};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
-use orchid_base::interner::{IStr, es, is};
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::iter_utils::IteratorPrint;
-use orchid_base::logging::log;
 use orchid_base::name::{NameLike, Sym, VName, VPath};
-use orchid_base::reqnot::{Client, ClientExt};
+use orchid_base::reqnot::{ReqNot, Requester};
-use orchid_base::stash::stash;
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
@@ -37,7 +35,7 @@ pub(crate) struct SystemInstData {
 	decl_id: api::SysDeclId,
 	lex_filter: api::CharFilter,
 	id: api::SysId,
-	line_types: Vec<IStr>,
+	line_types: Vec<Tok<String>>,
 	prelude: Vec<Sym>,
 	owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
 	pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
@@ -70,6 +68,8 @@ impl System {
 	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
 	#[must_use]
+	pub fn i(&self) -> &Interner { &self.0.ctx.i }
+	#[must_use]
 	pub fn deps(&self) -> &[System] { &self.0.deps }
 	#[must_use]
 	pub fn ctor(&self) -> SystemCtor {
@@ -77,27 +77,22 @@ impl System {
 			.expect("Ctor was used to create ext")
 	}
 	#[must_use]
-	pub(crate) fn client(&self) -> &dyn Client { self.0.ext.client() }
+	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { self.0.ext.reqnot() }
 	#[must_use]
 	pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
-		self.client().request(api::GetMember(self.0.id, id)).await.unwrap()
+		self.reqnot().request(api::GetMember(self.0.id, id)).await
 	}
 	#[must_use]
 	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
 	#[must_use]
-	pub fn can_lex(&self, c: char) -> bool {
+	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
-		let ret = char_filter_match(&self.0.lex_filter, c);
-		let ctor = self.ctor();
-		stash(async move { writeln!(log("debug"), "{} can lex {c}: {}", ctor.name(), ret).await });
-		ret
-	}
 	#[must_use]
 	pub fn prelude(&self) -> Vec<Sym> { self.0.prelude.clone() }
 	/// Have this system lex a part of the source. It is assumed that
 	/// [Self::can_lex] was called and returned true.
 	pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: IStr,
+		source: Tok<String>,
 		src: Sym,
 		pos: u32,
 		r: impl FnMut(u32) -> F,
@@ -105,16 +100,16 @@ impl System {
 		self.0.ext.lex_req(source, src, pos, self.id(), r).await
 	}
 	#[must_use]
-	pub fn get_parser(&self, ltyp: IStr) -> Option<Parser> {
+	pub fn get_parser(&self, ltyp: Tok<String>) -> Option<Parser> {
 		(self.0.line_types.iter().enumerate())
 			.find(|(_, txt)| *txt == &ltyp)
 			.map(|(idx, _)| Parser { idx: idx as u16, system: self.clone() })
 	}
-	pub fn line_types(&self) -> impl Iterator<Item = &IStr> + '_ { self.0.line_types.iter() }
+	pub fn line_types(&self) -> impl Iterator<Item = &Tok<String>> + '_ { self.0.line_types.iter() }
 
 	#[must_use]
 	pub async fn request(&self, req: Vec<u8>) -> Vec<u8> {
-		self.client().request(api::SysFwded(self.id(), req)).await.unwrap()
+		self.reqnot().request(api::SysFwded(self.id(), req)).await
 	}
 	pub(crate) async fn new_atom(&self, data: Vec<u8>, id: api::AtomId) -> AtomHand {
 		let mut owned_g = self.0.owned_atoms.write().await;
@@ -129,10 +124,10 @@ impl System {
 	}
 	pub(crate) fn drop_atom(&self, dropped_atom_id: api::AtomId) {
 		let this = self.0.clone();
-		let _ = self.0.ctx.spawn(Box::pin(async move {
+		(self.0.ctx.spawn)(Box::pin(async move {
-			this.ext.client().request(api::AtomDrop(this.id, dropped_atom_id)).await.unwrap();
+			this.ext.reqnot().request(api::AtomDrop(this.id, dropped_atom_id)).await;
 			this.owned_atoms.write().await.remove(&dropped_atom_id);
-		}));
+		}))
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakSystem {
@@ -142,7 +137,7 @@ impl System {
 	pub(crate) async fn name_resolver(
 		&self,
 		orig: api::ParsedConstId,
-	) -> impl AsyncFnMut(&[IStr]) -> OrcRes<VName> + use<> {
+	) -> impl AsyncFnMut(&[Tok<String>]) -> OrcRes<VName> + use<> {
 		let root = self.0.ctx.root.read().await.upgrade().expect("find_names when root not in context");
 		let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
 		let ctx = self.0.ctx.clone();
@@ -158,15 +153,14 @@ impl System {
 			}
 			match cmod.imports.get(selector) {
 				Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
-				Some(Err(dests)) => {
+				Some(Err(dests)) =>
 					return Err(mk_errv_floating(
-						is("Ambiguous name").await,
+						ctx.i.i("Ambiguous name").await,
 						format!(
 							"{selector} could refer to {}",
 							dests.iter().map(|ri| &ri.target).display("or")
 						),
-					));
+					)),
-				},
 				None => (),
 			}
 			if root_data.root.members.get(selector).is_some() {
@@ -176,7 +170,7 @@ impl System {
 				return Ok(VPath::new(cwd.iter().cloned()).name_with_suffix(selector.clone()));
 			}
 			Err(mk_errv_floating(
-				is("Invalid name").await,
+				ctx.i.i("Invalid name").await,
 				format!("{selector} doesn't refer to a module"),
 			))
 		}
@@ -209,7 +203,8 @@ impl SystemCtor {
 	#[must_use]
 	pub fn name(&self) -> &str { &self.decl.name }
 	pub async fn name_tok(&self) -> Sym {
-		(Sym::parse(&self.decl.name).await).expect("System cannot have empty name")
+		(Sym::parse(&self.decl.name, &self.ext.upgrade().expect("ext dropped early").ctx().i).await)
+			.expect("System cannot have empty name")
 	}
 	#[must_use]
 	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
@@ -225,17 +220,17 @@ impl SystemCtor {
 		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
 		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
 		let id = ext.ctx().next_sys_id();
-		let sys_inst =
+		let sys_inst = ext.reqnot().request(api::NewSystem { depends, id, system: self.decl.id }).await;
-			ext.client().request(api::NewSystem { depends, id, system: self.decl.id }).await.unwrap();
 		let data = System(Rc::new(SystemInstData {
 			deps,
 			decl_id: self.decl.id,
 			ext: ext.clone(),
 			ctx: ext.ctx().clone(),
 			lex_filter: sys_inst.lex_filter,
-			line_types: join_all(sys_inst.line_types.iter().map(|m| es(*m))).await,
+			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
+				.await,
 			id,
-			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok))).await,
+			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok, &ext.ctx().i))).await,
 			owned_atoms: RwLock::new(HashMap::new()),
 			const_paths: MemoMap::new(),
 		}));

orchid-host/src/tree.rs

@@ -13,11 +13,11 @@ use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use memo_map::MemoMap;
 use orchid_base::clone;
-use orchid_base::error::{OrcRes, mk_errv, report};
+use orchid_base::error::{OrcRes, Reporter, mk_errv};
-use orchid_base::interner::{IStr, IStrv, es, is, iv};
+use orchid_base::interner::Tok;
 use orchid_base::location::{CodeGenInfo, Pos};
 use orchid_base::name::{NameLike, Sym, VPath};
-use orchid_base::reqnot::ClientExt;
+use orchid_base::reqnot::Requester;
 
 use crate::api;
 use crate::ctx::Ctx;
@@ -45,7 +45,7 @@ impl Root {
 	#[must_use]
 	pub async fn from_api(api: api::Module, sys: &System) -> Self {
 		let consts = MemoMap::new();
-		let mut tfac = TreeFromApiCtx { consts: &consts, path: iv(&[][..]).await, sys };
+		let mut tfac = TreeFromApiCtx { consts: &consts, path: sys.i().i(&[][..]).await, sys };
 		let root = Module::from_api(api, &mut tfac).await;
 		Root(Rc::new(RwLock::new(RootData { root, consts, ctx: sys.ctx().clone() })))
 	}
@@ -60,10 +60,10 @@ impl Root {
 		Ok(Self(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() }))))
 	}
 	#[must_use]
-	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym) -> Self {
+	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym, rep: &Reporter) -> Self {
 		let mut ref_this = self.0.write().await;
 		let this = &mut *ref_this;
-		let mut deferred_consts = Vec::new();
+		let mut deferred_consts = HashMap::new();
 		let consts = this.consts.clone();
 		let mut tfpctx = FromParsedCtx {
 			pars_root: parsed,
@@ -72,6 +72,7 @@ impl Root {
 			pars_prefix: pars_prefix.clone(),
 			root: &this.root,
 			ctx: &this.ctx,
+			rep,
 		};
 		let mut module = Module::from_parsed(parsed, pars_prefix.clone(), &mut tfpctx).await;
 		for step in pars_prefix.iter().rev() {
@@ -86,9 +87,9 @@ impl Root {
 			.expect("Merge conflict between parsed and existing module");
 		let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
 		*this.ctx.root.write().await = new.downgrade();
-		for (path, sys_id, pc_id) in deferred_consts {
+		for (path, (sys_id, pc_id)) in deferred_consts {
 			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
-			let api_expr = sys.client().request(api::FetchParsedConst(sys.id(), pc_id)).await.unwrap();
+			let api_expr = sys.reqnot().request(api::FetchParsedConst(sys.id(), pc_id)).await;
 			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), this.ctx.clone()).await;
 			new.0.write().await.consts.insert(path, expr);
 		}
@@ -109,7 +110,7 @@ impl Root {
 		}
 		match module {
 			Ok(_) => Err(mk_errv(
-				is("module used as constant").await,
+				ctx.i.i("module used as constant").await,
 				format!("{name} is a module, not a constant"),
 				[pos],
 			)),
@@ -117,7 +118,7 @@ impl Root {
 				ChildErrorKind::Private => panic!("public_only is false"),
 				ChildErrorKind::Constant => panic!("Tree refers to constant not in table"),
 				ChildErrorKind::Missing => Err(mk_errv(
-					is("Constant does not exist").await,
+					ctx.i.i("Constant does not exist").await,
 					format!("{name} does not refer to a constant"),
 					[pos],
 				)),
@@ -143,12 +144,12 @@ impl Default for WeakRoot {
 pub struct TreeFromApiCtx<'a> {
 	pub sys: &'a System,
 	pub consts: &'a MemoMap<Sym, Expr>,
-	pub path: IStrv,
+	pub path: Tok<Vec<Tok<String>>>,
 }
 impl<'a> TreeFromApiCtx<'a> {
 	#[must_use]
-	pub async fn push<'c>(&'c self, name: IStr) -> TreeFromApiCtx<'c> {
+	pub async fn push<'c>(&'c self, name: Tok<String>) -> TreeFromApiCtx<'c> {
-		let path = iv(&self.path.iter().cloned().chain([name]).collect_vec()).await;
+		let path = self.sys.ctx().i.i(&self.path.iter().cloned().chain([name]).collect_vec()).await;
 		TreeFromApiCtx { path, consts: self.consts, sys: self.sys }
 	}
 }
@@ -161,17 +162,17 @@ pub struct ResolvedImport {
 
 #[derive(Clone, Default)]
 pub struct Module {
-	pub imports: HashMap<IStr, Result<ResolvedImport, Vec<ResolvedImport>>>,
+	pub imports: HashMap<Tok<String>, Result<ResolvedImport, Vec<ResolvedImport>>>,
-	pub members: HashMap<IStr, Rc<Member>>,
+	pub members: HashMap<Tok<String>, Rc<Member>>,
 }
 impl Module {
 	#[must_use]
 	pub async fn from_api(api: api::Module, ctx: &mut TreeFromApiCtx<'_>) -> Self {
 		let mut members = HashMap::new();
 		for mem in api.members {
-			let mem_name = es(mem.name).await;
+			let mem_name = ctx.sys.i().ex(mem.name).await;
 			let vname = VPath::new(ctx.path.iter().cloned()).name_with_suffix(mem_name.clone());
-			let name = vname.to_sym().await;
+			let name = vname.to_sym(ctx.sys.i()).await;
 			let (lazy, kind) = match mem.kind {
 				api::MemberKind::Lazy(id) =>
 					(Some(LazyMemberHandle { id, sys: ctx.sys.id(), path: name.clone() }), None),
@@ -204,23 +205,23 @@ impl Module {
 		let mut glob_imports_by_name = HashMap::<_, Vec<_>>::new();
 		for import in parsed.get_imports().into_iter().filter(|i| i.name.is_none()) {
 			let pos = import.sr.pos();
-			match absolute_path(&path, &import.path).await {
+			match absolute_path(&path, &import.path, &ctx.ctx.i).await {
-				Err(e) => report(e.err_obj(pos, &import.path.to_string()).await),
+				Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, pos, &import.path.to_string()).await),
 				Ok(abs_path) => {
 					let names_res = match abs_path.strip_prefix(&ctx.pars_prefix[..]) {
 						None => {
 							let mut tree_ctx = (ctx.ctx.clone(), ctx.consts);
-							resolv_glob(&path, ctx.root, &abs_path, pos, &mut tree_ctx).await
+							resolv_glob(&path, ctx.root, &abs_path, pos, &ctx.ctx.i, &mut tree_ctx).await
 						},
 						Some(sub_tgt) => {
 							let sub_path = (path.strip_prefix(&ctx.pars_prefix[..]))
 								.expect("from_parsed called with path outside pars_prefix");
-							resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &mut ()).await
+							resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &ctx.ctx.i, &mut ()).await
 						},
 					};
-					let abs_path = abs_path.to_sym().await;
+					let abs_path = abs_path.to_sym(&ctx.ctx.i).await;
 					match names_res {
-						Err(e) => report(e),
+						Err(e) => ctx.rep.report(e),
 						Ok(names) =>
 							for name in names {
 								match glob_imports_by_name.entry(name) {
@@ -243,28 +244,30 @@ impl Module {
 						prelude_item.last_seg(),
 						Ok(ResolvedImport {
 							target: prelude_item,
-							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude").await.pos(),
+							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude", &ctx.ctx.i)
+								.await
+								.pos(),
 						}),
 					);
 				}
 			}
 		}
-		let conflicting_imports_msg = is("Conflicting imports").await;
+		let conflicting_imports_msg = ctx.ctx.i.i("Conflicting imports").await;
 		for (key, values) in imports_by_name {
 			if values.len() == 1 {
 				let import = values.into_iter().next().unwrap();
 				let sr = import.sr.clone();
-				let abs_path_res = absolute_path(&path, &import.clone().mspath()).await;
+				let abs_path_res = absolute_path(&path, &import.clone().mspath(), &ctx.ctx.i).await;
 				match abs_path_res {
-					Err(e) => report(e.err_obj(sr.pos(), &import.to_string()).await),
+					Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, sr.pos(), &import.to_string()).await),
 					Ok(abs_path) => {
-						let target = abs_path.to_sym().await;
+						let target = abs_path.to_sym(&ctx.ctx.i).await;
 						imports.insert(key, Ok(ResolvedImport { target, pos: sr.pos() }));
 					},
 				}
 			} else {
 				for item in values {
-					report(mk_errv(
+					ctx.rep.report(mk_errv(
 						conflicting_imports_msg.clone(),
 						format!("{key} is imported multiple times from different modules"),
 						[item.sr.pos()],
@@ -274,11 +277,12 @@ impl Module {
 		}
 		for (key, values) in glob_imports_by_name {
 			if !imports.contains_key(&key) {
+				let i = &ctx.ctx.i;
 				let values = stream::iter(values)
 					.then(|(n, sr)| {
 						clone!(key; async move {
 							ResolvedImport {
-								target: n.to_vname().suffix([key.clone()]).to_sym().await,
+								target: n.to_vname().suffix([key.clone()]).to_sym(i).await,
 								pos: sr.pos(),
 							}
 						})
@@ -288,12 +292,12 @@ impl Module {
 				imports.insert(key, if values.len() == 1 { Ok(values[0].clone()) } else { Err(values) });
 			}
 		}
-		let self_referential_msg = is("Self-referential import").await;
+		let self_referential_msg = ctx.ctx.i.i("Self-referential import").await;
 		for (key, value) in imports.iter() {
 			let Ok(import) = value else { continue };
 			if import.target.strip_prefix(&path[..]).is_some_and(|t| t.starts_with(slice::from_ref(key)))
 			{
-				report(mk_errv(
+				ctx.rep.report(mk_errv(
 					self_referential_msg.clone(),
 					format!("import {} points to itself or a path within itself", &import.target),
 					[import.pos.clone()],
@@ -304,7 +308,7 @@ impl Module {
 		for item in &parsed.items {
 			match &item.kind {
 				ItemKind::Member(mem) => {
-					let path = path.to_vname().suffix([mem.name.clone()]).to_sym().await;
+					let path = path.to_vname().suffix([mem.name.clone()]).to_sym(&ctx.ctx.i).await;
 					let kind = OnceCell::from(MemberKind::from_parsed(&mem.kind, path.clone(), ctx).await);
 					members.insert(
 						mem.name.clone(),
@@ -381,16 +385,17 @@ pub struct FromParsedCtx<'a> {
 	pars_prefix: Sym,
 	pars_root: &'a ParsedModule,
 	root: &'a Module,
+	rep: &'a Reporter,
 	ctx: &'a Ctx,
 	consts: &'a MemoMap<Sym, Expr>,
-	deferred_consts: &'a mut Vec<(Sym, api::SysId, api::ParsedConstId)>,
+	deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
 }
 
 impl Tree for Module {
 	type Ctx<'a> = (Ctx, &'a MemoMap<Sym, Expr>);
 	async fn child(
 		&self,
-		key: IStr,
+		key: Tok<String>,
 		public_only: bool,
 		(ctx, consts): &mut Self::Ctx<'_>,
 	) -> crate::dealias::ChildResult<'_, Self> {
@@ -405,7 +410,7 @@ impl Tree for Module {
 			MemberKind::Const => Err(ChildErrorKind::Constant),
 		}
 	}
-	fn children(&self, public_only: bool) -> hashbrown::HashSet<IStr> {
+	fn children(&self, public_only: bool) -> hashbrown::HashSet<Tok<String>> {
 		self.members.iter().filter(|(_, v)| !public_only || v.public).map(|(k, _)| k.clone()).collect()
 	}
 }
@@ -435,7 +440,7 @@ impl MemberKind {
 	async fn from_parsed(parsed: &ParsedMemberKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
 		match parsed {
 			ParsedMemberKind::Const(id, sys) => {
-				ctx.deferred_consts.push((path, sys.id(), *id));
+				ctx.deferred_consts.insert(path, (sys.id(), *id));
 				MemberKind::Const
 			},
 			ParsedMemberKind::Mod(m) =>

orchid-std/Cargo.toml

@@ -3,20 +3,11 @@ name = "orchid-std"
 version = "0.1.0"
 edition = "2024"
 
-[[bin]]
-name = "orchid-std-piped"
-path = "src/main.rs"
-
-[lib]
-crate-type = ["cdylib", "lib"]
-name = "orchid_std"
-path = "src/lib.rs"
-
 [dependencies]
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
 futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.16.1"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 never = "0.1.0"
 once_cell = "1.21.3"
@@ -27,13 +18,12 @@ orchid-base = { version = "0.1.0", path = "../orchid-base" }
 orchid-extension = { version = "0.1.0", path = "../orchid-extension", features = [
 	"tokio",
 ] }
-ordered-float = "5.1.0"
+ordered-float = "5.0.0"
-pastey = "0.2.1"
+pastey = "0.1.1"
-rust_decimal = "1.39.0"
+rust_decimal = "1.38.0"
 subslice-offset = "0.1.1"
 substack = "1.1.1"
-tokio = { version = "1.49.0", features = ["full"] }
+tokio = { version = "1.47.1", features = ["full"] }
-unicode-segmentation = "1.12.0"
 
 [dev-dependencies]
-test_executors = "0.4.1"
+test_executors = "0.3.5"