Generate Typed Jerboa Rust crates
ober
68131a69aec434cdb641668877f08a20f627815c
--- a/Makefile +++ b/Makefile @@ -27,8 +27,10 @@ CHEZ_EXT_LDPATH = $(CHEZ_EXT_DIR)/chez-ssl:$(CHEZ_EXT_DIR)/chez-zlib:$(CHEZ_EXT_ PURE_AUDIT_ROOT ?= $(HOME)/mine PURE_AUDIT_ARGS ?= --summary --discover $(PURE_AUDIT_ROOT) TYPED_SOURCES ?= tests/fixtures/typed/valid-split-tree.ss +TYPED_RUST_SOURCES ?= tests/fixtures/typed/rust-basic.ss +TYPED_RUST_DIR ?= build/typed/rust -.PHONY: help chez chez-cross build binary binary-cross native-cross pure-audit typecheck typed-test typed-clean test test-reader test-core test-runtime test-stdlib test-ffi test-modules test-expanded test-contract test-ergo test-typed-parser test-typed-checker test-typed-rust test-pure-audit test-features test-wrappers test-phase4a test-phase4b test-phase4c test-phase4d test-phase4e test-phase4f test-phase5 test-phase5e test-phase6 test-phase7 test-phase8 test-functional test-repl test-security test-security-profile test-native test-gaps native clean-native audit-native clean security security-production security-profile fuzz fuzz-smoke fuzz-deep fuzz-reader-fuzz fuzz-json-fuzz fuzz-http2-fuzz fuzz-dns-fuzz fuzz-pregexp-fuzz fuzz-csv-fuzz fuzz-base64-fuzz fuzz-hex-fuzz fuzz-uri-fuzz fuzz-format-fuzz fuzz-router-fuzz fuzz-sandbox-fuzz test-rawstring test-regex test-rx test-peg test-regex-all check-docs check-docs-strict docker-build docker-push +.PHONY: help chez chez-cross build binary binary-cross native-cross pure-audit typecheck typed-rust typed-build typed-test typed-clean test test-reader test-core test-runtime test-stdlib test-ffi test-modules test-expanded test-contract test-ergo test-typed-parser test-typed-checker test-typed-rust test-pure-audit test-features test-wrappers test-phase4a test-phase4b test-phase4c test-phase4d test-phase4e test-phase4f test-phase5 test-phase5e test-phase6 test-phase7 test-phase8 test-functional test-repl test-security test-security-profile test-native test-gaps native clean-native audit-native clean security security-production security-profile fuzz fuzz-smoke fuzz-deep fuzz-reader-fuzz fuzz-json-fuzz fuzz-http2-fuzz fuzz-dns-fuzz fuzz-pregexp-fuzz fuzz-csv-fuzz fuzz-base64-fuzz fuzz-hex-fuzz fuzz-uri-fuzz fuzz-format-fuzz fuzz-router-fuzz fuzz-sandbox-fuzz test-rawstring test-regex test-rx test-peg test-regex-all check-docs check-docs-strict docker-build docker-push help: @echo "Usage: make <target>" @@ -43,6 +45,8 @@ help: @echo " native-cross Cross-build Rust native lib for a target" @echo " Args: CHEZ_TARGET_MACHINE=<mt> CROSS_CC=<cc> [CROSS_NATIVE_FEATURES=tls,...]" @echo " typecheck Typecheck Typed Jerboa sources" + @echo " typed-rust Generate Rust crate sources from Typed Jerboa" + @echo " typed-build Generate Rust crate sources and run cargo check" @echo " typed-test Run Typed Jerboa front-end tests" @echo " typed-clean Remove generated typed build artifacts" @echo " native Build Rust native library" @@ -304,6 +308,12 @@ test: test-reader test-core test-runtime test-stdlib test-ffi test-modules test- typecheck: @$(SCHEME) --libdirs $(LIBDIRS) --script support/typecheck.ss $(TYPED_SOURCES) +typed-rust: + @$(SCHEME) --libdirs $(LIBDIRS) --script support/typed-rust.ss $(TYPED_RUST_DIR) $(TYPED_RUST_SOURCES) + +typed-build: typed-rust + @cargo check --manifest-path $(TYPED_RUST_DIR)/Cargo.toml + typed-test: test-typed-parser test-typed-checker test-typed-rust typecheck typed-clean: --- a/docs/jerboa-to-rust.md +++ b/docs/jerboa-to-rust.md @@ -430,18 +430,20 @@ Add targets: ```make typed-rust -typed-rust-check -typed-rust-build -typed-rust-test -typed-rust-clean +typed-build +typed-test +typed-clean ``` -Suggested behavior: +Current and suggested behavior: -- `typed-rust-check`: generate Rust and run `cargo check` -- `typed-rust-build`: compile Rust artifact -- `typed-rust-test`: run Rust unit tests plus Jerboa boundary tests -- `typed-rust-clean`: remove generated Rust build outputs +- `typed-rust`: generate `build/typed/rust/Cargo.toml` and + `build/typed/rust/src/lib.rs` +- `typed-build`: generate Rust and run `cargo check`; later this should compile + the native artifact too +- `typed-test`: run parser, checker, emitter, and typecheck tests; later add + Rust unit tests plus Jerboa boundary tests +- `typed-clean`: remove generated Rust build outputs Use Cargo for MVP. Add direct `rustc` mode later if static build integration needs tighter control. @@ -538,12 +540,12 @@ Second module: typed `rope`. - Generate deterministic Rust text. Initial `(jerboa typed rust)` emitter landed with safe Rust headers, identifier sanitization, primitive type mapping, records, variants, and primitive function bodies. -- Generate crate directory. -- Generate `Cargo.toml`. -- Generate `lib.rs`. +- Generate crate directory. Initial `build/typed/rust` generation landed. +- Generate `Cargo.toml`. Initial disposable crate manifest landed. +- Generate `lib.rs`. Initial single-file crate output landed. - Generate one function returning an integer. Initial text emission for `Nat` - functions landed; crate compilation is next. -- Build with Cargo. + functions landed. +- Build with Cargo. Initial `make typed-build` target runs `cargo check`. - Call from Jerboa through FFI. ### Milestone 2: Primitive Types --- a/docs/typed-jerboa.md +++ b/docs/typed-jerboa.md @@ -136,7 +136,8 @@ The compiler can later accept `.tss` as a convenience. Current landing: - `(jerboa typed parser)` parses and validates `(typed-library ...)` datums - into a first AST layer. + into a first AST layer. It strips Jerboa reader source annotations so file + based `typecheck` and backend generation use the same AST path as unit tests. - `(jerboa typed checker)` performs the first backend-neutral validation pass: duplicate names, export resolution, type-reference resolution, compound type arities, duplicate fields, duplicate params, duplicate variant cases, and a @@ -147,8 +148,11 @@ Current landing: and a short hint for common errors. - `(jerboa typed rust)` emits deterministic safe Rust text for checked typed modules. The first emitter covers Rust identifiers, primitive types, - records, variants, and primitive function bodies; crate generation and FFI - wrappers are still future work. + records, variants, and primitive function bodies; native artifact builds and + FFI wrappers are still future work. +- `support/typed-rust.ss`, `make typed-rust`, and `make typed-build` generate a + disposable Cargo crate under `build/typed/rust`; `typed-build` runs + `cargo check` against the generated crate. - The parser currently recognizes explicit `export` forms, `type` declarations, immutable and `mut` record fields, variants including nullary cases, and `def` forms with typed parameters and `:` or `->` return markers. @@ -628,9 +632,10 @@ Initial meanings: - `typecheck`: parse and type check typed modules. Initial target landed for parser/checker validation over `TYPED_SOURCES`. -- `typed-rust`: generate Rust for typed modules. Initial Rust text emission has - landed as `(jerboa typed rust)`; the public make target is still pending. -- `typed-build`: generate Rust, compile artifacts, generate wrappers. +- `typed-rust`: generate Rust for typed modules. Initial target writes + `build/typed/rust/Cargo.toml` and `build/typed/rust/src/lib.rs`. +- `typed-build`: generate Rust and run `cargo check`. Native artifact builds + and wrappers are still pending. - `typed-test`: run typed compiler tests and runtime boundary tests. Initial target runs parser, checker, Rust emitter, and `typecheck`. - `typed-llvm`: reserved until the LLVM backend exists. @@ -777,6 +782,8 @@ Minimum excluded features: - Generate Rust structs and enums. Initial safe Rust text emission landed. - Generate Rust functions. Initial primitive expression function emission landed. +- Generate a Cargo crate. Initial `make typed-rust` / `make typed-build` + targets landed for a primitive typed fixture. - Generate conversion functions. - Compile generated Rust as a static or dynamic library. - Generate Jerboa wrappers. --- a/lib/jerboa/typed/parser.ss +++ b/lib/jerboa/typed/parser.ss @@ -8,6 +8,7 @@ (library (jerboa typed parser) (export typed-library-form? + strip-source-annotations parse-typed-library parse-typed-declaration parse-typed-type @@ -37,7 +38,8 @@ typed-def-name typed-def-params typed-def-return-type typed-def-body) (import (chezscheme) ; jerboa-security: suppress direct-chezscheme-import-user-code -- trusted typed compiler front-end module - (only (jerboa core) def defstruct)) + (only (jerboa core) def defstruct) + (jerboa reader)) (defstruct typed-module (name exports declarations)) (defstruct typed-type-decl (name)) @@ -48,6 +50,18 @@ (defstruct typed-param (name type)) (defstruct typed-def (name params return-type body)) + (def (strip-source-annotations datum) + (cond + [(annotated-datum? datum) + (strip-source-annotations (annotated-datum-value datum))] + [(pair? datum) + (cons (strip-source-annotations (car datum)) + (strip-source-annotations (cdr datum)))] + [(vector? datum) + (list->vector + (map strip-source-annotations (vector->list datum)))] + [else datum])) + (def (proper-list? x) (cond [(null? x) #t] @@ -95,31 +109,34 @@ exports)) (def (typed-library-form? form) - (and (pair? form) - (eq? (car form) 'typed-library))) + (let ([form (strip-source-annotations form)]) + (and (pair? form) + (eq? (car form) 'typed-library)))) (def (parse-typed-library form) - (expect-proper-list 'parse-typed-library form) - (unless (typed-library-form? form) - (error 'parse-typed-library "expected typed-library form" form)) - (unless (>= (length form) 3) - (error 'parse-typed-library - "expected (typed-library name (export ...) declarations ...)" - form)) - (let ([name (parse-module-name (cadr form))] - [exports (parse-export-form (caddr form))] - [declarations (map parse-typed-declaration (cdddr form))]) - (make-typed-module name exports declarations))) + (let ([form (strip-source-annotations form)]) + (expect-proper-list 'parse-typed-library form) + (unless (typed-library-form? form) + (error 'parse-typed-library "expected typed-library form" form)) + (unless (>= (length form) 3) + (error 'parse-typed-library + "expected (typed-library name (export ...) declarations ...)" + form)) + (let ([name (parse-module-name (cadr form))] + [exports (parse-export-form (caddr form))] + [declarations (map parse-typed-declaration (cdddr form))]) + (make-typed-module name exports declarations)))) (def (parse-typed-type type) - (cond - [(symbol? type) type] - [(and (pair? type) (proper-list? type) (symbol? (car type))) - (cons (car type) (map parse-typed-type (cdr type)))] - [else - (error 'parse-typed-type - "expected a type symbol or compound type expression" - type)])) + (let ([type (strip-source-annotations type)]) + (cond + [(symbol? type) type] + [(and (pair? type) (proper-list? type) (symbol? (car type))) + (cons (car type) (map parse-typed-type (cdr type)))] + [else + (error 'parse-typed-type + "expected a type symbol or compound type expression" + type)]))) (def (parse-field form) (expect-proper-list 'parse-typed-field form) @@ -220,17 +237,18 @@ (expect-symbol 'parse-typed-type-decl (cadr form) form))) (def (parse-typed-declaration form) - (expect-proper-list 'parse-typed-declaration form) - (unless (pair? form) - (error 'parse-typed-declaration "empty declaration" form)) - (case (car form) - [(type) (parse-type-decl form)] - [(record) (parse-record form)] - [(variant) (parse-variant form)] - [(def) (parse-def form)] - [else - (error 'parse-typed-declaration - "unsupported typed declaration" - form)])) + (let ([form (strip-source-annotations form)]) + (expect-proper-list 'parse-typed-declaration form) + (unless (pair? form) + (error 'parse-typed-declaration "empty declaration" form)) + (case (car form) + [(type) (parse-type-decl form)] + [(record) (parse-record form)] + [(variant) (parse-variant form)] + [(def) (parse-def form)] + [else + (error 'parse-typed-declaration + "unsupported typed declaration" + form)]))) ) ;; end library --- a/lib/jerboa/typed/rust.ss +++ b/lib/jerboa/typed/rust.ss @@ -9,7 +9,9 @@ (export rust-symbol-name typed-module->rust-string - typed-library-form->rust-string) + typed-library-form->rust-string + typed-modules->rust-crate-string + typed-library-forms->rust-crate-string) (import (chezscheme) ; jerboa-security: suppress direct-chezscheme-import-user-code -- trusted typed compiler Rust emitter (only (jerboa core) def) @@ -333,15 +335,22 @@ [(typed-def? decl) (emit-def decl port)] [else #f])) - (def (emit-module module port) + (def (emit-rust-header port) (write-line port 0 "// Generated by Jerboa's typed Rust backend. Do not edit.") (write-line port 0 "#![forbid(unsafe_code)]") - (newline port) + (write-line port 0 "#![allow(unused_parens)]") + (newline port)) + + (def (emit-module-declarations module port) (for-each (lambda (decl) (emit-declaration decl port)) (typed-module-declarations module))) + (def (emit-module module port) + (emit-rust-header port) + (emit-module-declarations module port)) + (def (typed-module->rust-string module) (let ([errors (check-typed-module module)]) (unless (null? errors) @@ -355,4 +364,24 @@ (def (typed-library-form->rust-string form) (typed-module->rust-string (parse-typed-library form))) + (def (typed-modules->rust-crate-string modules) + (for-each + (lambda (module) + (let ([errors (check-typed-module module)]) + (unless (null? errors) + (error 'typed-modules->rust-crate-string + "typed module has check errors" + (map typed-check-error-kind errors))))) + modules) + (emit-to-string + (lambda (port) + (emit-rust-header port) + (for-each + (lambda (module) + (emit-module-declarations module port)) + modules)))) + + (def (typed-library-forms->rust-crate-string forms) + (typed-modules->rust-crate-string (map parse-typed-library forms))) + ) ;; end library --- a/support/typecheck.ss +++ b/support/typecheck.ss @@ -47,11 +47,15 @@ exn)) 1]) (let ([forms (typed-library-forms (read-all-forms path))]) - (let loop ([rest forms] [count 0]) - (if (null? rest) - count - (loop (cdr rest) - (+ count (check-typed-form path (car rest))))))))) + (if (null? forms) + (begin + (printf "~a: no typed-library forms found\n" path) + 1) + (let loop ([rest forms] [count 0]) + (if (null? rest) + count + (loop (cdr rest) + (+ count (check-typed-form path (car rest)))))))))) (define args (command-line-arguments)) new file mode 100644 --- /dev/null +++ b/support/typed-rust.ss @@ -0,0 +1,126 @@ +#!chezscheme +;;; typed-rust.ss -- generate a Rust crate from Typed Jerboa source files +;;; +;;; Usage: +;;; scheme --libdirs lib --script support/typed-rust.ss OUT-DIR file.ss ... + +(import (chezscheme) ; jerboa-security: suppress direct-chezscheme-import-user-code -- trusted typed Rust generator script + (jerboa reader) + (jerboa typed parser) + (jerboa typed rust)) + +(define (usage) + (display "Usage: scheme --libdirs lib --script support/typed-rust.ss OUT-DIR file.ss ...\n")) + +(define (string-split-slash text) + (let ([len (string-length text)]) + (let loop ([i 0] [start 0] [out '()]) + (cond + [(= i len) + (let ([part (substring text start i)]) + (reverse (if (string=? part "") out (cons part out))))] + [(char=? (string-ref text i) #\/) + (let ([part (substring text start i)]) + (loop (+ i 1) + (+ i 1) + (if (string=? part "") out (cons part out))))] + [else (loop (+ i 1) start out)])))) + +(define (absolute-path? path) + (and (> (string-length path) 0) + (char=? (string-ref path 0) #\/))) + +(define (path-join2 base part) + (cond + [(string=? base "") part] + [(string=? base "/") (string-append "/" part)] + [else (string-append base "/" part)])) + +(define (path-has-parent-reference? path) + (let loop ([parts (string-split-slash path)]) + (cond + [(null? parts) #f] + [(string=? (car parts) "..") #t] + [else (loop (cdr parts))]))) + +(define (validate-output-path path) + (when (or (absolute-path? path) + (path-has-parent-reference? path)) + (error 'typed-rust + "output paths must be relative and must not contain parent directory references" + path)) + path) + +(define (ensure-directory path) + (cond + [(file-directory? path) #t] + [(file-exists? path) + (error 'typed-rust "path exists but is not a directory" path)] + [else + (mkdir path) + #t])) + +(define (ensure-directory-tree path) + (let ([parts (string-split-slash path)] + [root (if (absolute-path? path) "/" "")]) + (let loop ([rest parts] [current root]) + (unless (null? rest) + (let ([next (path-join2 current (car rest))]) + (ensure-directory next) + (loop (cdr rest) next)))))) + +(define (write-file-string path text) + (let ([safe-path (validate-output-path path)] + [port #f]) + (dynamic-wind + (lambda () + (set! port (open-output-file safe-path 'replace))) + (lambda () (display text port)) + (lambda () + (when port + (close-port port)))))) + +(define (typed-library-forms forms) + (let loop ([rest forms] [out '()]) + (cond + [(null? rest) (reverse out)] + [(typed-library-form? (car rest)) + (loop (cdr rest) (cons (car rest) out))] + [else + (loop (cdr rest) out)]))) + +(define (read-typed-library-forms path) + (typed-library-forms (jerboa-read-file path))) + +(define (read-all-typed-library-forms paths) + (let loop ([rest paths] [out '()]) + (if (null? rest) + (reverse out) + (loop (cdr rest) + (append (reverse (read-typed-library-forms (car rest))) out))))) + +(define cargo-toml + "[package]\nname = \"jerboa-typed-generated\"\nversion = \"0.0.0\"\nedition = \"2021\"\npublish = false\n\n[lib]\ncrate-type = [\"rlib\"]\n") + +(define (generate-rust-crate out-dir source-paths) + (let* ([safe-out-dir (validate-output-path out-dir)] + [src-dir (path-join2 safe-out-dir "src")] + [forms (read-all-typed-library-forms source-paths)]) + (when (null? forms) + (error 'typed-rust "no typed-library forms found" source-paths)) + (let ([rust (typed-library-forms->rust-crate-string forms)]) + (ensure-directory-tree src-dir) + (write-file-string (path-join2 safe-out-dir "Cargo.toml") cargo-toml) + (write-file-string (path-join2 src-dir "lib.rs") rust) + (printf "Typed Jerboa Rust: wrote ~a module~a to ~a\n" + (length forms) + (if (= (length forms) 1) "" "s") + safe-out-dir)))) + +(define args (command-line-arguments)) + +(when (< (length args) 2) + (usage) + (exit 2)) + +(generate-rust-crate (car args) (cdr args)) new file mode 100644 --- /dev/null +++ b/tests/fixtures/typed/rust-basic.ss @@ -0,0 +1,21 @@ +(typed-library (sample typed rust-basic) + (export zero add-one positive? choose greeting double-add) + + (def (zero) : Nat + 0) + + (def (add-one (x : Nat)) : Nat + (+ x 1)) + + (def (positive? (x : Nat)) : Bool + (> x 0)) + + (def (choose (flag : Bool)) : Nat + (if flag 1 0)) + + (def (greeting) : String + "hello") + + (def (double-add (x : Nat)) : Nat + (let ((y (+ x x))) + (+ y 1)))) --- a/tests/test-typed-parser.ss +++ b/tests/test-typed-parser.ss @@ -2,6 +2,7 @@ ;;; Tests for (jerboa typed parser) (import (chezscheme) + (jerboa reader) (jerboa typed parser)) (define pass 0) @@ -51,6 +52,8 @@ 0))) (define parsed (parse-typed-library sample)) +(define annotated-sample + (car (jerboa-read-file "tests/fixtures/typed/rust-basic.ss"))) (define declarations (typed-module-declarations parsed)) (define type-decl (list-ref declarations 0)) (define record-decl (list-ref declarations 1)) @@ -68,6 +71,14 @@ (typed-library-form? '(library (x))) #f) +(test "typed-library-form? reader annotation" + (typed-library-form? annotated-sample) + #t) + +(test "parse reader-annotated typed library" + (typed-module-name (parse-typed-library annotated-sample)) + '(sample typed rust-basic)) + (test "module name" (typed-module-name parsed) '(sample typed split-tree)) --- a/tests/test-typed-rust.ss +++ b/tests/test-typed-rust.ss @@ -29,7 +29,7 @@ (+ x 1)))) (define calc-rust - "// Generated by Jerboa's typed Rust backend. Do not edit.\n#![forbid(unsafe_code)]\n\npub fn zero() -> u64 {\n 0u64\n}\n\npub fn add_one(x: u64) -> u64 {\n (x + 1u64)\n}\n\n") + "// Generated by Jerboa's typed Rust backend. Do not edit.\n#![forbid(unsafe_code)]\n#![allow(unused_parens)]\n\npub fn zero() -> u64 {\n 0u64\n}\n\npub fn add_one(x: u64) -> u64 {\n (x + 1u64)\n}\n\n") (define data-form '(typed-library (sample typed data) @@ -43,7 +43,7 @@ (Noop)))) (define data-rust - "// Generated by Jerboa's typed Rust backend. Do not edit.\n#![forbid(unsafe_code)]\n\n#[derive(Clone, Debug, PartialEq)]\npub struct Pane {\n pub id: u64,\n pub focused_p: bool,\n}\n\n#[derive(Clone, Debug, PartialEq)]\npub enum EditOp {\n Insert {\n at: u64,\n text: String,\n },\n Noop,\n}\n\n") + "// Generated by Jerboa's typed Rust backend. Do not edit.\n#![forbid(unsafe_code)]\n#![allow(unused_parens)]\n\n#[derive(Clone, Debug, PartialEq)]\npub struct Pane {\n pub id: u64,\n pub focused_p: bool,\n}\n\n#[derive(Clone, Debug, PartialEq)]\npub enum EditOp {\n Insert {\n at: u64,\n text: String,\n },\n Noop,\n}\n\n") (printf "--- Typed Jerboa Rust emitter tests ---~%")