Split typed Rust crate into per-module files
ober
a01b132215a33d6d3976e4016f7d742393cd53ae
--- a/docs/jerboa-to-rust.md +++ b/docs/jerboa-to-rust.md @@ -112,25 +112,30 @@ Done in subsequent phases: bodies, every `let`, `if`, and `match` IR node also emits an inline `/* source: path:line:column */` block comment so the compiler-reported Rust line can be mapped back to typed source. +- Per-module file layout. The crate emitter now writes a thin `src/lib.rs` + that `pub mod`s each module and `pub use`s its items at the crate root, plus + one `src/<module>.rs` per typed module containing that module's records, + variants, resources, defs, and ABI wrappers. Runtime helpers + (`jt_return_bytes`, `jt_clone_handle`, panic capture, the handle registry) + live in `lib.rs` with `pub(crate)` visibility so per-module files can + reach them via `use crate::*;`. Cross-module bare-name calls keep working + because of the crate-root `pub use` re-exports. Still open: -- Per-module file layout (`src/modules/<module>.rs`). The current emitter - inlines all modules into a single `lib.rs`. - Direct `rustc` / static binary integration polish and eventual LLVM parity tests. Today the build goes through `cargo build` only. -- Cross-module `use` declarations and per-module Rust files for richer - workspaces. +- Explicit per-module `use` declarations driven by the typed `import` graph + (today every module imports `crate::*;`). ## Output Layout -Candidate layout: +Current layout: ```text build/typed/rust/Cargo.toml -build/typed/rust/src/lib.rs -build/typed/rust/src/modules/<module>.rs -build/typed/rust/src/runtime.rs +build/typed/rust/src/lib.rs # mod decls + re-exports + runtime helpers +build/typed/rust/src/<module>.rs # one file per typed module build/typed/rust/target/... build/typed/jerboa/<module>.ss ``` @@ -572,8 +577,9 @@ typed-clean Current and suggested behavior: -- `typed-rust`: generate `build/typed/rust/Cargo.toml` and - `build/typed/rust/src/lib.rs` +- `typed-rust`: generate `build/typed/rust/Cargo.toml`, + `build/typed/rust/src/lib.rs`, and one + `build/typed/rust/src/<module>.rs` per typed module - `typed-wrappers`: generate `.ss` wrapper files under `build/typed/jerboa` - `typed-build`: generate Rust and run `cargo build`, producing disposable `rlib`, `staticlib`, and `cdylib` artifacts plus wrapper files --- a/lib/jerboa/typed/rust.ss +++ b/lib/jerboa/typed/rust.ss @@ -14,7 +14,9 @@ typed-module->rust-string typed-library-form->rust-string typed-modules->rust-crate-string - typed-library-forms->rust-crate-string) + typed-modules->rust-crate-files + typed-library-forms->rust-crate-string + typed-library-forms->rust-crate-files) (import (chezscheme) ; jerboa-security: suppress direct-chezscheme-import-user-code -- trusted typed compiler Rust emitter (only (jerboa core) def) @@ -615,11 +617,11 @@ (write-line port 0 "static JT_HANDLES: OnceLock<Mutex<HashMap<u64, std::boxed::Box<dyn Any + Send>>>> = OnceLock::new();") (write-line port 0 "static JT_NEXT_HANDLE: AtomicU64 = AtomicU64::new(1);") (newline port) - (write-line port 0 "fn jt_handles() -> &'static Mutex<HashMap<u64, std::boxed::Box<dyn Any + Send>>> {") + (write-line port 0 "pub(crate) fn jt_handles() -> &'static Mutex<HashMap<u64, std::boxed::Box<dyn Any + Send>>> {") (write-line port 1 "JT_HANDLES.get_or_init(|| Mutex::new(HashMap::new()))") (write-line port 0 "}") (newline port) - (write-line port 0 "fn jt_store_handle<T: Any + Send>(value: T) -> u64 {") + (write-line port 0 "pub(crate) fn jt_store_handle<T: Any + Send>(value: T) -> u64 {") (write-line port 1 "let id = JT_NEXT_HANDLE.fetch_add(1, Ordering::Relaxed);") (write-line port 1 "if let Ok(mut handles) = jt_handles().lock() {") (write-line port 2 "handles.insert(id, std::boxed::Box::new(value));") @@ -629,7 +631,7 @@ (write-line port 1 "}") (write-line port 0 "}") (newline port) - (write-line port 0 "fn jt_clone_handle<T: Any + Clone>(id: u64) -> Option<T> {") + (write-line port 0 "pub(crate) fn jt_clone_handle<T: Any + Clone>(id: u64) -> Option<T> {") (write-line port 1 "let handles = jt_handles().lock().ok()?;") (write-line port 1 "handles.get(&id).and_then(|value| value.downcast_ref::<T>().cloned())") (write-line port 0 "}") @@ -651,7 +653,7 @@ (write-line port 0 "}") (newline port) (write-line port 0 - "fn jt_capture_panic(payload: std::boxed::Box<dyn std::any::Any + Send>) {") + "pub(crate) fn jt_capture_panic(payload: std::boxed::Box<dyn std::any::Any + Send>) {") (write-line port 1 "let msg = if let Some(s) = payload.downcast_ref::<&'static str>() {") (write-line port 2 "(*s).to_string()") (write-line port 1 "} else if let Some(s) = payload.downcast_ref::<String>() {") @@ -679,7 +681,7 @@ (newline port)) (def (emit-byte-buffer-runtime port) - (write-line port 0 "fn jt_return_bytes(bytes: Vec<u8>, out_ptr: *mut *mut u8, out_len: *mut usize) -> bool {") + (write-line port 0 "pub(crate) fn jt_return_bytes(bytes: Vec<u8>, out_ptr: *mut *mut u8, out_len: *mut usize) -> bool {") (write-line port 1 "if out_ptr.is_null() || out_len.is_null() {") (write-line port 2 "return false;") (write-line port 1 "}") @@ -1680,36 +1682,87 @@ (def (typed-library-form->rust-string form) (typed-module->rust-string (parse-typed-library form))) - (def (typed-modules->rust-crate-string modules) - ;; Run import-aware elaboration so cross-module references type check. - (let* ([results (check-and-elaborate-typed-modules modules)] - [ir-envs - (map (lambda (entry) - (let ([modname (car entry)] - [errors (cadr entry)] - [defs (caddr entry)]) - (unless (null? errors) - (error 'typed-modules->rust-crate-string - "typed module has check errors" - (cons modname (map typed-check-error-kind errors)))) - (map (lambda (ed) - (cons (elaborated-def-name ed) - (elaborated-def-body-ir ed))) - defs))) - results)]) + (def (elaborate-modules-or-error modules who) + (let ([results (check-and-elaborate-typed-modules modules)]) + (map (lambda (entry) + (let ([modname (car entry)] + [errors (cadr entry)] + [defs (caddr entry)]) + (unless (null? errors) + (error who + "typed module has check errors" + (cons modname (map typed-check-error-kind errors)))) + (map (lambda (ed) + (cons (elaborated-def-name ed) + (elaborated-def-body-ir ed))) + defs))) + results))) + + (def (emit-lib-rs modules port) + (emit-rust-header port) + (for-each + (lambda (module) + (write-line port 0 + (string-append "pub mod " (module-abi-prefix module) ";"))) + modules) + (unless (null? modules) (newline port)) + (for-each + (lambda (module) + (write-line port 0 + (string-append "pub use " (module-abi-prefix module) "::*;"))) + modules) + (unless (null? modules) (newline port)) + (emit-runtime-helpers modules port)) + + (def (emit-module-file module ir-env port) + (write-line port 0 "// Generated by Jerboa's typed Rust backend. Do not edit.") + (write-line port 0 "#![allow(unused_imports)]") + (newline port) + (write-line port 0 "use crate::*;") + (newline port) + (parameterize ([*rust-ir-env* ir-env]) + (emit-module-declarations module port))) + + (def (typed-modules->rust-crate-files modules) + ;; Returns list of (relative-path . content) pairs describing the crate's + ;; src/ tree: a thin src/lib.rs plus one src/<module>.rs per typed module. + (let ([ir-envs (elaborate-modules-or-error modules + 'typed-modules->rust-crate-files)]) (with-rust-env modules (lambda () - (emit-to-string - (lambda (port) - (emit-rust-header port) - (emit-runtime-helpers modules port) - (let loop ([rest-modules modules] [rest-envs ir-envs]) - (cond - [(null? rest-modules) #f] - [else - (parameterize ([*rust-ir-env* (car rest-envs)]) - (emit-module-declarations (car rest-modules) port)) - (loop (cdr rest-modules) (cdr rest-envs))])))))))) + (let* ([lib-content + (emit-to-string + (lambda (port) (emit-lib-rs modules port)))] + [module-files + (let loop ([rest modules] [envs ir-envs] [out '()]) + (cond + [(null? rest) (reverse out)] + [else + (let* ([m (car rest)] + [name (module-abi-prefix m)] + [path (string-append "src/" name ".rs")] + [content + (emit-to-string + (lambda (port) + (emit-module-file m (car envs) port)))]) + (loop (cdr rest) (cdr envs) + (cons (cons path content) out)))]))]) + (cons (cons "src/lib.rs" lib-content) module-files)))))) + + (def (typed-modules->rust-crate-string modules) + ;; Concatenation of every generated crate file. Retained so existing + ;; substring-based tests (and any callers that just want one blob of text) + ;; keep working after the move to per-module files. + (let ([files (typed-modules->rust-crate-files modules)]) + (let loop ([rest files] [port (open-output-string)]) + (cond + [(null? rest) (get-output-string port)] + [else + (display (cdar rest) port) + (loop (cdr rest) port)])))) + + (def (typed-library-forms->rust-crate-files forms) + (typed-modules->rust-crate-files (map parse-typed-library forms))) (def (typed-library-forms->rust-crate-string forms) (typed-modules->rust-crate-string (map parse-typed-library forms))) --- a/support/typed-rust.ss +++ b/support/typed-rust.ss @@ -102,19 +102,38 @@ (define cargo-toml "[package]\nname = \"jerboa-typed-generated\"\nversion = \"0.0.0\"\nedition = \"2021\"\npublish = false\n\n[lib]\ncrate-type = [\"rlib\", \"staticlib\", \"cdylib\"]\n") +(define (path-parent path) + (let loop ([i (- (string-length path) 1)]) + (cond + [(< i 0) ""] + [(char=? (string-ref path i) #\/) + (substring path 0 i)] + [else (loop (- i 1))]))) + +(define (write-crate-file out-dir entry) + (let* ([rel-path (car entry)] + [content (cdr entry)] + [full-path (path-join2 out-dir rel-path)] + [parent (path-parent full-path)]) + (unless (string=? parent "") + (ensure-directory-tree parent)) + (write-file-string full-path content))) + (define (generate-rust-crate out-dir source-paths) (let* ([safe-out-dir (validate-output-path out-dir)] [src-dir (path-join2 safe-out-dir "src")] [forms (read-all-typed-library-forms source-paths)]) (when (null? forms) (error 'typed-rust "no typed-library forms found" source-paths)) - (let ([rust (typed-library-forms->rust-crate-string forms)]) + (let ([files (typed-library-forms->rust-crate-files forms)]) (ensure-directory-tree src-dir) (write-file-string (path-join2 safe-out-dir "Cargo.toml") cargo-toml) - (write-file-string (path-join2 src-dir "lib.rs") rust) - (printf "Typed Jerboa Rust: wrote ~a module~a to ~a\n" + (for-each (lambda (entry) (write-crate-file safe-out-dir entry)) files) + (printf "Typed Jerboa Rust: wrote ~a module~a (~a file~a) to ~a\n" (length forms) (if (= (length forms) 1) "" "s") + (length files) + (if (= (length files) 1) "" "s") safe-out-dir)))) (define args (command-line-arguments)) --- a/tests/test-typed-rust.ss +++ b/tests/test-typed-rust.ss @@ -35,7 +35,7 @@ (define safe-prelude (string-append - "fn jt_return_bytes(bytes: Vec<u8>, out_ptr: *mut *mut u8, out_len: *mut usize) -> bool {\n" + "pub(crate) fn jt_return_bytes(bytes: Vec<u8>, out_ptr: *mut *mut u8, out_len: *mut usize) -> bool {\n" " if out_ptr.is_null() || out_len.is_null() {\n" " return false;\n" " }\n" @@ -62,7 +62,7 @@ "thread_local! {\n" " static JT_LAST_PANIC: std::cell::RefCell<Option<String>> = const { std::cell::RefCell::new(None) };\n" "}\n\n" - "fn jt_capture_panic(payload: std::boxed::Box<dyn std::any::Any + Send>) {\n" + "pub(crate) fn jt_capture_panic(payload: std::boxed::Box<dyn std::any::Any + Send>) {\n" " let msg = if let Some(s) = payload.downcast_ref::<&'static str>() {\n" " (*s).to_string()\n" " } else if let Some(s) = payload.downcast_ref::<String>() {\n" @@ -590,6 +590,56 @@ (substring? annotated-control-rust "*/ {"))) #t) +(define crate-files + (typed-library-forms->rust-crate-files + (list import-provider-form import-consumer-form))) + +(define (assoc-string key files) + (let loop ([rest files]) + (cond + [(null? rest) #f] + [(string=? (caar rest) key) (cdar rest)] + [else (loop (cdr rest))]))) + +(test "rust crate yields lib.rs as first file" + (caar crate-files) + "src/lib.rs") + +(test "rust crate yields one file per typed module" + (length crate-files) + 3) + +(test "rust crate lib.rs declares each module" + (let ([lib (assoc-string "src/lib.rs" crate-files)]) + (and (substring? lib "pub mod rust_import_provider;") + (substring? lib "pub mod rust_import_consumer;") + (substring? lib "pub use rust_import_provider::*;") + (substring? lib "pub use rust_import_consumer::*;"))) + #t) + +(test "rust crate provider module file holds provider defs" + (let ([provider (assoc-string "src/rust_import_provider.rs" crate-files)]) + (and (substring? provider "use crate::*;") + (substring? provider "pub fn inc(x: u64) -> u64") + (substring? provider "pub struct Pt {"))) + #t) + +(test "rust crate consumer module file calls into provider via bare names" + (let ([consumer (assoc-string "src/rust_import_consumer.rs" crate-files)]) + (and (substring? consumer "use crate::*;") + (substring? consumer "pub fn bump_pair_x(p: Pt) -> Pt") + (substring? consumer "inc("))) + #t) + +(test "rust crate runtime helpers live only in lib.rs" + (let ([lib (assoc-string "src/lib.rs" crate-files)] + [provider (assoc-string "src/rust_import_provider.rs" crate-files)] + [consumer (assoc-string "src/rust_import_consumer.rs" crate-files)]) + (and (substring? lib "pub(crate) fn jt_return_bytes") + (not (substring? provider "fn jt_return_bytes")) + (not (substring? consumer "fn jt_return_bytes")))) + #t) + (printf "~%Typed Rust emitter: ~a passed, ~a failed~%" pass fail) (when (> fail 0) (exit 1))