Improve local model verified repair workflow
ober
d5f3c6881898bf61b29963aecda9cee08478d6b5
--- a/docs/rle-benchmark-optimization.md +++ b/docs/rle-benchmark-optimization.md @@ -36,11 +36,21 @@ compliance. ## Current Outcome -The retained implementation passes 1,166 tests with zero failures and one -environment-dependent image-backend skip. Provider-level local integration now -proves that an explicit primary escalation reaches a separately configured -expert, preserves the bounded help note and active tool protocol, returns a -structured edit call, and records both usage events. +The retained implementation passes 1,203 tests with zero failures and one +environment-dependent image-backend skip. The final installed binary has +separate local-origin behavior: bounded inspection, strict transactional edits, +sticky expert escalation after repeated rejection, an 8,192-token first-draft +cap, and a 4,096-token cap after the first successful edit or verifier failure. +The local-origin cap follows a handoff to a cloud expert; ordinary cloud-origin +workflows remain unchanged. + +The latest local DeepSeek V4 Pro hard-RLE run is documented under **Local +DeepSeek V4 Pro Iteration (2026-07-15)** below. It improved the earlier local +hard result from a globally broken 0/22 artifact to 20/22 after 971 seconds. +A continuation on the promoted artifact reached 21/22 before the combined +30-minute budget expired. This is substantial progress, but not a pass. + +### Earlier OpenRouter Cohort Live exact-model scoring after the early successful cohorts is blocked by the OpenRouter account balance: every current GLM 5.2, DeepSeek V4 Pro, Kimi K2.7 @@ -61,6 +71,40 @@ task, expert mode, and the 72-turn default. The archived `command.txt` files do not contain `--no-expert`. Each stopped in about one second at the account-level 402 before producing a model turn or workspace diff. +## Local DeepSeek V4 Pro Iteration (2026-07-15) + +Primary artifact: + +```text +~/mine/kratistos/runs/20260715-212140-life-rle-local-ds4-jcode-expertcap +``` + +The cold run used `mlx-ds4:deepseek-v4-pro`, expert mode, and the hard RLE task. +The local model made four bounded repository inspections, then generated +several malformed full drafts. Jcode kept those drafts off disk. After repeated +same-path rejection it switched to the configured OpenRouter DeepSeek expert; +both local and expert requests were capped at 8,192 completion tokens. + +The seventh draft contained a mismatched closer. The deterministic delimiter +repair replaced that closer, required the complete file to balance and remain +readable Scheme, and promoted a 335-line `life-rle.ss`. The run was manually +interrupted at 971 seconds just after promotion, before its next verifier turn. +Running `make test` directly on the artifact produced 20/22: optional header +whitespace and row-height overflow were the only failures. + +A continuation on that exact artifact used verifier output and exact replacement +edits. It fixed header detection and row-height overflow handling, reaching +21/22. The remaining failure was spaced rule syntax (`B3 / S23`). The combined +cold run and continuation exhausted the intended 30-minute budget, so the final +expert turn was stopped and the result remains a fail. + +This trajectory identified the final retained optimization: 8K is useful while +a local model may need to emit the first complete implementation, but excessive +for a localized repair. Post-edit and post-verifier local-origin turns now use a +4K cap, including expert handoffs. Deterministic tests verify the 8K-to-4K +transition; a new full cold benchmark was not run after this last timing-only +change. + ## Experiment Rules 1. Change one workflow behavior at a time. --- a/src/jcode/core/expert.ss +++ b/src/jcode/core/expert.ss @@ -31,6 +31,7 @@ expert-handoff-instruction chat-with-expert chat-with-expert-via-stream + chat-direct-via-stream stream-chat-with-expert current-expert-cb) @@ -322,3 +323,18 @@ provider messages tools (lambda (_token) (void))))) (record-current-usage! usage) (make-assistant-message content tool-calls))) + +(def (chat-direct-via-stream provider messages tools) + "Call PROVIDER directly over the streaming transport and preserve usage. + Verified workflows use this for an explicit one-shot expert handoff after + repeated verifier failures; unlike chat-with-expert-via-stream, this does + not run escalation detection recursively." + (let-values (((content tool-calls usage) + (provider-stream-chat + provider + (with-expert-handoff + messages "repeated verifier failures" "") + tools + (lambda (_token) (void))))) + (record-current-usage! usage) + (make-assistant-message content tool-calls))) --- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -31,6 +31,7 @@ :jcode/core/expert :jcode/core/errors :jcode/tool/registry + :jcode/provider/provider :jcode/proxy/server) (def default-verify-command "make build") @@ -38,6 +39,10 @@ (def default-verified-max-tool-errors 5) (def default-verified-history-token-limit 48000) (def default-verified-history-keep-batches 3) +(def default-local-verified-history-token-limit 24000) +(def default-local-verified-history-keep-batches 2) +(def default-local-verified-max-completion-tokens 8192) +(def default-local-verified-repair-max-completion-tokens 4096) (def (opt-get o key) (let ((p (assoc key o))) (and p (cdr p)))) @@ -165,6 +170,18 @@ (def (verified-shell-tool-spec? spec) (member (tool-spec-name spec) '("run" "bash" "shell"))) +(def (verified-inspection-tool-spec? spec) + (or (verified-shell-tool-spec? spec) + (member (tool-spec-name spec) + '("read" "list" "ls" "cat" "head" "tail" "wc" "balance")) + (verified-mcp-tool-spec? spec))) + +(def (verified-staged-repair-tool-spec? spec) + (or (member (tool-spec-name spec) + '("line_edit" "replace_def" "replace_range")) + (and (current-serving-forced-expert?) + (member (tool-spec-name spec) '("edit" "write"))))) + (def (verified-post-edit-tool-spec? spec) (member (tool-spec-name spec) '("edit" "write" "line_edit" "replace_def" "replace_range" @@ -202,6 +219,44 @@ (>= (current-pre-edit-run-alias-count) pre-edit-run-alias-limit)))) #f) + ;; Local inference pays heavily for every exploratory round. Once the + ;; shared inspection budget is spent, expose only actions that can + ;; create the first draft and reach the verifier. + ((and (current-verified-local-model?) + (= (current-successful-edit-count) 0) + (not (current-rejected-ss-draft)) + (not (current-pending-ss-create-repair)) + (>= (current-pre-edit-inspection-count) + local-pre-edit-inspection-limit) + (verified-inspection-tool-spec? spec)) + #f) + ;; A local model that has consumed its focused post-verifier reads + ;; should not be offered another inspection tool that will only be + ;; rejected. Keep edits and verify visible so the next turn makes + ;; progress. + ((and (current-verified-local-model?) + (current-after-failed-verify?) + (>= (current-inspections-after-failed-verify) + local-inspection-after-failed-verify-limit) + (verified-inspection-tool-spec? spec)) + #f) + ;; The verifier already includes bounded Jerboa diagnostics. Local + ;; models do better spending their two repair observations on the + ;; implicated source than opening another MCP round-trip. + ((and (current-verified-local-model?) + (current-after-failed-verify?) + (verified-mcp-tool-spec? spec)) + #f) + ;; After repeated rejected drafts, the latest candidate is retained + ;; in staging and its diagnostic is already in history. Offer only + ;; bounded staged edits; another read or full rewrite repeats the + ;; expensive failure cycle. + ((and (current-verified-local-model?) + (current-rejected-ss-draft) + (> (current-rejected-draft-inspections) + rejected-draft-inspection-limit) + (not (verified-staged-repair-tool-spec? spec))) + #f) ((not (verified-mcp-tool-spec? spec)) #t) (else (cond @@ -228,9 +283,36 @@ (make-provider-backend provider chat-with-expert-via-stream)))) (lambda (messages tool-specs step) - (if (procedure? provider) - (backend messages (verified-provider-tool-specs tool-specs) step) - (backend messages (verified-provider-tool-specs tool-specs) #f))))) + (parameterize + ((current-max-tokens-cap + (if (and (current-verified-local-model?) + (or (current-after-failed-verify?) + (> (current-successful-edit-count) 0))) + default-local-verified-repair-max-completion-tokens + (current-max-tokens-cap)))) + (if (procedure? provider) + (backend messages (verified-provider-tool-specs tool-specs) step) + (let* ((sticky-rejected-draft? + (and (current-verified-local-model?) + (current-rejected-ss-draft) + (> (current-rejected-draft-inspections) + rejected-draft-inspection-limit) + (config-expert-enabled?))) + (forced-expert? + (and (config-expert-enabled?) + (or sticky-rejected-draft? + (current-force-expert-next?)))) + (specs + (parameterize + ((current-serving-forced-expert? forced-expert?)) + (verified-provider-tool-specs tool-specs)))) + (if forced-expert? + (begin + (current-force-expert-next? #f) + ((make-provider-backend + (get-expert-provider) chat-direct-via-stream) + messages specs #f)) + (backend messages specs #f)))))))) ;; ── verify oracle ────────────────────────────────────────────────────── (def (tail-lines s n) @@ -766,6 +848,19 @@ tool-name ". Stop using shell-shaped exploration. The next useful tool must be read/list on a specific current-repo path if one file is still unknown, or edit/write with the first complete draft, then verify(). Do not probe environment paths, HOME, PATH, or external installs."))))) +(def (local-pre-edit-inspection-block-message tool-name) + (and (current-verified-local-model?) + (= (current-successful-edit-count) 0) + (not (current-rejected-ss-draft)) + (not (current-pending-ss-create-repair)) + (let ((next (+ (current-pre-edit-inspection-count) 1))) + (current-pre-edit-inspection-count next) + (and (> next local-pre-edit-inspection-limit) + (string-append + "local-model pre-edit inspection limit reached while calling " + (tool-label tool-name) + ". The repository and task already provide enough context. Stop inspecting external files and write the first complete draft now; then call verify. Inspection tools are hidden until the first edit."))))) + (def (post-failed-verify-mcp-block-message name) (and (current-after-failed-verify?) (not (verified-targeted-after-failure-mcp-tool? name)) @@ -798,6 +893,8 @@ (make-tool-spec name desc params) (lambda (args) (or (rejected-draft-hard-recovery-message cwd name) + (let ((msg (local-pre-edit-inspection-block-message name))) + (and msg (raise-recoverable-tool-error msg name))) (let ((msg (pre-edit-mcp-block-message name))) (and msg (raise-recoverable-tool-error msg name))) (let ((msg (post-failed-verify-mcp-block-message name))) @@ -986,6 +1083,21 @@ (def current-pre-edit-run-alias-count (make-parameter 0)) +(def current-pre-edit-inspection-count + (make-parameter 0)) + +(def current-verified-local-model? + (make-parameter #f)) + +(def current-verify-failure-count + (make-parameter 0)) + +(def current-force-expert-next? + (make-parameter #f)) + +(def current-serving-forced-expert? + (make-parameter #f)) + (def current-last-verify-detail (make-parameter #f)) @@ -1019,6 +1131,9 @@ (current-successful-edit-count 0) (current-pre-edit-mcp-count 0) (current-pre-edit-run-alias-count 0) + (current-pre-edit-inspection-count 0) + (current-verify-failure-count 0) + (current-force-expert-next? #f) (current-last-verify-detail #f) (current-required-range-repair #f) (current-required-repair-inspections 0)) @@ -1031,6 +1146,14 @@ (def pre-edit-run-alias-limit 4) +(def local-pre-edit-inspection-limit 6) + +(def local-inspection-after-failed-verify-limit 2) + +(def local-expert-after-verify-failures 2) + +(def local-expert-after-rejected-drafts 2) + (def rejected-draft-inspection-limit 2) (def rejected-draft-repeat-threshold 1) @@ -1150,18 +1273,30 @@ (current-rejected-draft-path-reject-count path-reject-count) (current-rejected-draft-needs-inspection (and same-path? (>= path-reject-count 2))) - (if same? - (let ((n (+ (current-rejected-draft-repeat-count) 1))) - (current-rejected-draft-repeat-count n) - (when (>= n rejected-draft-repeat-threshold) - (current-rejected-draft-inspections - (max (current-rejected-draft-inspections) - (+ rejected-draft-inspection-limit 1)))) - n) - (begin - (current-rejected-draft-repeat-count 0) - (current-rejected-draft-inspections 0) - 0)))) + (let ((repeat-result + (if same? + (let ((n (+ (current-rejected-draft-repeat-count) 1))) + (current-rejected-draft-repeat-count n) + (when (>= n rejected-draft-repeat-threshold) + (current-rejected-draft-inspections + (max (current-rejected-draft-inspections) + (+ rejected-draft-inspection-limit 1)))) + n) + (begin + (current-rejected-draft-repeat-count 0) + (current-rejected-draft-inspections 0) + 0)))) + (when (and (current-verified-local-model?) + same-path? + (>= path-reject-count local-expert-after-rejected-drafts)) + ;; The latest full tool arguments and syntax diagnostic are in history, + ;; so the expert can patch staging without another broad inspection. + (current-rejected-draft-inspections + (+ rejected-draft-inspection-limit 1)) + (current-rejected-draft-needs-inspection #f) + (when (config-expert-enabled?) + (current-force-expert-next? #t))) + repeat-result))) (def (rejected-draft-repeat-note path) (and (>= (current-rejected-draft-repeat-count) @@ -1214,7 +1349,18 @@ (current-required-repair-inspections 0)) (def (record-verify-result! result . review-after-verify) - (current-after-failed-verify? (not (and (pair? result) (car result)))) + (let ((passed? (and (pair? result) (car result)))) + (current-after-failed-verify? (not passed?)) + (if passed? + (begin + (current-verify-failure-count 0) + (current-force-expert-next? #f)) + (let ((failures (+ (current-verify-failure-count) 1))) + (current-verify-failure-count failures) + (when (and (current-verified-local-model?) + (>= failures local-expert-after-verify-failures) + (config-expert-enabled?)) + (current-force-expert-next? #t))))) (current-requirements-review-pending? (and (pair? result) (car result) @@ -1254,9 +1400,12 @@ (def (note-inspection-after-failed-verify! who) (if (current-after-failed-verify?) - (let ((n (+ (current-inspections-after-failed-verify) 1))) + (let ((n (+ (current-inspections-after-failed-verify) 1)) + (limit (if (current-verified-local-model?) + local-inspection-after-failed-verify-limit + inspection-after-failed-verify-limit))) (current-inspections-after-failed-verify n) - (if (> n inspection-after-failed-verify-limit) + (if (> n limit) (inspection-limit-message who) #f)) #f)) @@ -1500,7 +1649,9 @@ (repair-ref repair 'end)))) (string-append msg - "\nThe file was not written. Structural repair is still pending; do not switch to edit or a full-file rewrite. " + (if (current-verified-local-model?) + "\nThe file was not written. Structural repair is still pending. Retry with a span that leaves the complete file syntax-correct, or use edit/write with a syntax-correct complete file when this span exposes a second structural error. " + "\nThe file was not written. Structural repair is still pending; do not switch to edit or a full-file rewrite. ") (range-repair-instruction repair) (best-repair-candidate-text cwd repair label))) (string-append msg "\n" (ss-repair-instruction path))))) @@ -1942,12 +2093,18 @@ (def (do-read args cwd) (cond + ((local-pre-edit-inspection-block-message 'read) + => (lambda (msg) (raise-recoverable-tool-error msg 'read))) ((rejected-draft-hard-recovery-message cwd 'read) => (lambda (msg) (raise-recoverable-tool-error msg 'read))) + ((and (current-verified-local-model?) + (note-inspection-after-failed-verify! 'read)) + => (lambda (msg) (raise-recoverable-tool-error msg 'read))) ((required-repair-read-block-message cwd args) => (lambda (msg) (raise-recoverable-tool-error msg 'read))) ((current-required-range-repair) (do-read-current args cwd)) - ((note-inspection-after-failed-verify! 'read) + ((and (not (current-verified-local-model?)) + (note-inspection-after-failed-verify! 'read)) => (lambda (msg) (raise-recoverable-tool-error msg 'read))) ((note-inspection-after-edit! 'read) => (lambda (msg) (raise-recoverable-tool-error msg 'read))) @@ -1955,11 +2112,17 @@ (def (do-list args cwd) (cond + ((local-pre-edit-inspection-block-message 'list) + => (lambda (msg) (raise-recoverable-tool-error msg 'list))) ((rejected-draft-hard-recovery-message cwd 'list) => (lambda (msg) (raise-recoverable-tool-error msg 'list))) + ((and (current-verified-local-model?) + (note-inspection-after-failed-verify! 'list)) + => (lambda (msg) (raise-recoverable-tool-error msg 'list))) ((required-repair-tool-block-message cwd 'list args) => (lambda (msg) (raise-recoverable-tool-error msg 'list))) - ((note-inspection-after-failed-verify! 'list) + ((and (not (current-verified-local-model?)) + (note-inspection-after-failed-verify! 'list)) => (lambda (msg) (raise-recoverable-tool-error msg 'list))) ((note-inspection-after-edit! 'list) => (lambda (msg) (raise-recoverable-tool-error msg 'list))) @@ -2314,6 +2477,69 @@ (and (balance-ok-text? (balance-report line-candidate label)) line-candidate))))))) +(def (line-column-index content target-line target-col) + (let ((n (string-length content))) + (let loop ((i 0) (line 1) (col 1)) + (cond + ((>= i n) #f) + ((and (= line target-line) (= col target-col)) i) + ((char=? (string-ref content i) #\newline) + (loop (+ i 1) (+ line 1) 1)) + (else (loop (+ i 1) line (+ col 1))))))) + +(def (matching-close-delim opener) + (cond + ((char=? opener #\() #\)) + ((char=? opener #\[) #\]) + ((char=? opener #\{) #\}) + (else #f))) + +(def (replace-char-at content index replacement) + (string-append + (substring content 0 index) + (string replacement) + (substring content (+ index 1) (string-length content)))) + +(def (repair-one-mismatched-close content label) + (let* ((report (balance-report content label)) + (marker "Mismatched close ") + (marker-at (string-contains report marker)) + (line (and marker-at + (find-line-number-after report " at line "))) + (col (and marker-at + (find-line-number-after report ", column "))) + (opener-marker "top opener is ") + (opener-at (and marker-at + (string-contains report opener-marker))) + (opener-index (and opener-at + (+ opener-at (string-length opener-marker)))) + (opener (and opener-index + (< opener-index (string-length report)) + (string-ref report opener-index))) + (expected (and opener (matching-close-delim opener))) + (index (and line col + (line-column-index content line col)))) + (and expected index + (replace-char-at content index expected)))) + +(def mismatched-close-auto-repair-limit 4) + +(def (mismatched-close-candidate content label) + (let loop ((candidate content) (left mismatched-close-auto-repair-limit) + (changed? #f)) + (let ((report (balance-report candidate label))) + (cond + ((balance-ok-text? report) (and changed? candidate)) + ((<= left 0) #f) + ((string-prefix? "Mismatched close " report) + (let ((next (repair-one-mismatched-close candidate label))) + (and next (loop next (- left 1) #t)))) + (changed? + ;; A corrected mismatch can expose a safe missing suffix. Reuse the + ;; existing conservative EOF repair rather than guessing at a span. + (minimal-balance-candidate candidate label)) + (else #f))))) + (def (read-all-forms content) (guard (e [else #f]) (with-input-from-string content @@ -2635,6 +2861,9 @@ (def (balance-file args cwd) (let ((path (arg-path args #f))) (cond + ((and (current-verified-local-model?) + (note-inspection-after-failed-verify! 'balance)) + => (lambda (msg) (raise-recoverable-tool-error msg 'balance))) ((not path) (let ((repair (current-required-range-repair))) (if repair @@ -2959,7 +3188,8 @@ (cond ((pre-edit-run-alias-block-message "run") => (lambda (msg) (raise-recoverable-tool-error msg 'run))) - ((note-inspection-after-failed-verify! 'run) + ((and (not (current-verified-local-model?)) + (note-inspection-after-failed-verify! 'run)) => (lambda (msg) (raise-recoverable-tool-error msg 'run))) ((and (string? cmd) (not (null? words)) @@ -3300,11 +3530,19 @@ ;; Keep heredoc-like content out of the lightweight delimiter scanner. (not (string-contains content "#<<")) (not (balance-ok-text? (balance-report content path))) - (let ((candidate (minimal-balance-candidate content path))) + (let ((candidate + (or (mismatched-close-candidate content path) + (minimal-balance-candidate content path)))) (and candidate (autoclose-candidate-syntax-sane? candidate path) candidate)))) +(def (auto-balance-repair-message path original-content) + (if (string-prefix? "Mismatched close " + (balance-report original-content path)) + " after applying a minimal delimiter repair" + " after appending minimal delimiter suffix")) + (def (jerboa-syntax-guard-message path content) (and (source-ss-path? path) (parameterize ((current-guard-content content)) @@ -3450,7 +3688,8 @@ (let ((repair (current-required-range-repair)) (msg (jerboa-syntax-guard-message path full-content))) (when msg - (if (and repair + (if (and (not (current-verified-local-model?)) + repair (repair-path-matches? cwd repair path) (or (generated-required-repair-content? cwd repair replacement-content) (and (delimiter-balance-guard-message? msg) @@ -3825,7 +4064,7 @@ (number->string (string-length final-content)) " bytes)" (if balanced-content - " after appending minimal delimiter suffix" + (auto-balance-repair-message path new-content) ""))))))) ((not (file-exists? p)) (raise-recoverable-tool-error @@ -3903,7 +4142,7 @@ (number->string (string-length final-content)) " bytes)" (if balanced-content - " after appending minimal delimiter suffix" + (auto-balance-repair-message path content) ""))))))))) ) @@ -4218,6 +4457,7 @@ (fn args))))) (run-aliases? (let ((p (assoc 'run-aliases? o))) (if p (cdr p) #f))) + (local-model? (and (opt-get o 'local-model?) #t)) (external-tools? (let ((p (assoc 'external-tools? o))) (if p (cdr p) #t))) (compact? (or (let ((p (assoc 'compact? o))) @@ -4472,6 +4712,14 @@ (if compact? "Compact verified mode: inspect only the few files needed to remove uncertainty, then write a complete first version and call verify. Before the first write, prefer read/list on the current repo or JCODE_READ_ROOTS over broad MCP/API discovery. After verify fails, repair the concrete verifier error with the smallest edit and verify again.\n" "") + (if local-model? + (string-append + "Local-model execution: all read/list/shell/MCP inspection shares a hard pre-edit budget of " + (number->string local-pre-edit-inspection-limit) + " calls. Do not inspect external Jerboa sources unless one exact API fact blocks the implementation. Write the first complete draft within this budget. After a failed verify, use at most " + (number->string local-inspection-after-failed-verify-limit) + " focused inspections before editing the diagnosed span. Local replace_range edits are transactional: a syntactically broken result is rejected and leaves the on-disk file unchanged.\n") + "") (if run-aliases? (string-append "run/bash/shell are narrow inspection aliases only; use verify for the configured build/test command. " @@ -4511,6 +4759,7 @@ ;; (MaxIterations / StepEnforcement / ToolExecution / NoProgress) on failure. (def (verified-run provider task . opt) (let* ((o (if (pair? opt) (car opt) '())) + (local-model? (and (opt-get o 'local-model?) #t)) (vcmd (or (opt-get o 'verify-command) default-verify-command)) (cwd (or (opt-get o 'cwd) ".")) (k (or (opt-get o 'best-of) 1)) @@ -4527,6 +4776,7 @@ (cons 'run-aliases? (let ((p (assoc 'run-aliases? o))) (if p (cdr p) #f))) + (cons 'local-model? local-model?) (cons 'task-guidance task-guidance) (cons 'compact? (and (opt-get o 'compact?) #t)) (cons 'review-after-verify? @@ -4547,9 +4797,13 @@ (compact-verified-history messages (or (opt-get o 'history-token-limit) - default-verified-history-token-limit) + (if local-model? + default-local-verified-history-token-limit + default-verified-history-token-limit)) (or (opt-get o 'history-keep-batches) - default-verified-history-keep-batches)))) + (if local-model? + default-local-verified-history-keep-batches + default-verified-history-keep-batches))))) (cons 'on-message (opt-get o 'on-message))))) (parameterize ((current-write-scope scope) (current-pending-ss-create-repair #f) @@ -4575,6 +4829,14 @@ (current-successful-edit-count 0) (current-pre-edit-mcp-count 0) (current-pre-edit-run-alias-count 0) + (current-pre-edit-inspection-count 0) + (current-verified-local-model? local-model?) + (current-max-tokens-cap + (and local-model? + default-local-verified-max-completion-tokens)) + (current-verify-failure-count 0) + (current-force-expert-next? #f) + (current-serving-forced-expert? #f) (current-last-verify-detail #f) (current-required-range-repair #f) (current-required-repair-inspections 0)) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -11,6 +11,8 @@ provider-name provider-model provider-base-url + provider-local? + current-max-tokens-cap current-stream-abort? model-rejects-tools? extract-text-tool-calls @@ -136,6 +138,33 @@ (def (provider-model p) (provider-record-model p)) (def (provider-base-url p) (provider-record-base-url p)) +(def (private-base-url? url) + (and (string? url) + (or (string-prefix? "http://localhost" url) + (string-prefix? "https://localhost" url) + (string-prefix? "http://127." url) + (string-prefix? "https://127." url) + (string-prefix? "http://[::1]" url) + (string-prefix? "https://[::1]" url) + (string-prefix? "http://10." url) + (string-prefix? "https://10." url) + (string-prefix? "http://192.168." url) + (string-prefix? "https://192.168." url) + (let loop ((octet 16)) + (and (<= octet 31) + (or (string-prefix? + (format "http://172.~a." octet) url) + (string-prefix? + (format "https://172.~a." octet) url) + (loop (+ octet 1)))))))) + +(def (provider-local? provider) + "Treat built-in local providers and loopback/private-network endpoints as + local. The endpoint check keeps custom aliases from silently receiving the + slower cloud workflow." + (or (local-provider? (provider-name provider)) + (private-base-url? (provider-base-url provider)))) + (def make-provider (case-lambda ((name api-key model) @@ -954,17 +983,25 @@ ((string? v) (positive-int-value (string->number v))) (else #f))) +;; Verified local workflows bind this to keep any primary or expert repair +;; response from consuming most of a wall-clock budget. Ordinary chat and +;; cloud-origin workflows leave it unbound and are unaffected. +(def current-max-tokens-cap (make-parameter #f)) + (def (openai-max-tokens provider) - (or (positive-int-value (getenv "JCODE_MAX_TOKENS")) - (positive-int-value - (config-ref "providers" (provider-name provider) "max_tokens")) - (positive-int-value - (config-ref "providers" (provider-name provider) "max_completion_tokens")) - (positive-int-value (config-ref "max_tokens")) - (positive-int-value (config-ref "max_completion_tokens")) - (if (local-provider? (provider-name provider)) - *openai-local-default-max-tokens* - *openai-default-max-tokens*))) + (let* ((configured + (or (positive-int-value (getenv "JCODE_MAX_TOKENS")) + (positive-int-value + (config-ref "providers" (provider-name provider) "max_tokens")) + (positive-int-value + (config-ref "providers" (provider-name provider) "max_completion_tokens")) + (positive-int-value (config-ref "max_tokens")) + (positive-int-value (config-ref "max_completion_tokens")) + (if (local-provider? (provider-name provider)) + *openai-local-default-max-tokens* + *openai-default-max-tokens*))) + (cap (positive-int-value (current-max-tokens-cap)))) + (if cap (min configured cap) configured))) (def (decimal-at s start) (let ((n (string-length s))) --- a/src/jcode/tool/verified.ss +++ b/src/jcode/tool/verified.ss @@ -6,7 +6,9 @@ :jcode/core/agent :jcode/core/agent-defs :jcode/core/log + :jcode/core/models :jcode/core/verified-run + :jcode/provider/provider :jcode/tool/registry) (def (init-verified-tool) @@ -74,6 +76,8 @@ (cons 'verify-command vcmd) (cons 'cwd cwd) (cons 'write-scope scope) + (cons 'local-model? + (provider-local? (get-current-provider))) (cons 'max-iterations iters))))) (string-append "VERIFIED RUN PASSED\n" --- a/src/jcode/ui/cli.ss +++ b/src/jcode/ui/cli.ss @@ -870,6 +870,8 @@ EXAMPLES: (cons 'cwd work-cwd) (cons 'write-scope scope) (cons 'run-aliases? run-aliases?) + (cons 'local-model? + (provider-local? provider)) ;; A focused MCP menu lowers prompt cost and tool ;; choice ambiguity while retaining Jerboa recipes, ;; syntax checks, and targeted API lookup. --- a/test/run.ss +++ b/test/run.ss @@ -2726,6 +2726,196 @@ "blocked until the rejected draft is inspected")))]))))) (safe-delete-test-file! target-path)) + (let* ([vr-dir "/tmp"] + [target "jcode-verified-local-inspection-budget.txt"] + [target-path (string-append vr-dir "/" target)] + [initial "one\ntwo\nthree\nfour\nfive\nsix\nseven\n"] + [tool-results '()] + [slurp (lambda (p) + (call-with-input-file p (lambda (i) (get-string-all i))))]) + (safe-delete-test-file! target-path) + (write-test-output-file target-path + (lambda (o) (display initial o)) 'replace) + (let* ([read-call + (lambda (start) + (list + (make-wtool-call "read" + (list (cons "path" target) + (cons "start" start) + (cons "end" start)) + #f)))] + [resp + (scripted-responder + (append + (map read-call '(1 2 3 4 5 6 7)) + (list + (list + (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" "fixed\n")) + #f)) + (list (make-wtool-call "verify" '() #f)))))] + [result + (verified-run resp "stop local exploration and edit" + (list + (cons 'cwd vr-dir) + (cons 'verify-command + (string-append "grep -q fixed " target)) + (cons 'write-scope (parse-write-scope target)) + (cons 'local-model? #t) + (cons 'max-iterations 12) + (cons 'max-tool-errors 0) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results)))))))]) + (check! "verified-run: local inspection budget still reaches verify" + result "VERIFIED: exit 0\n") + (check! "verified-run: local inspection budget permits the first edit" + (slurp target-path) "fixed\n") + (check-pred! "verified-run: local inspection budget refuses the seventh read" + (reverse tool-results) + (lambda (xs) + (let loop ([ys xs]) + (cond + [(null? ys) #f] + [(and (str-contains? (car ys) "[ToolRecoverableError]") + (str-contains? (car ys) + "local-model pre-edit inspection limit reached")) #t] + [else (loop (cdr ys))]))))) + (safe-delete-test-file! target-path)) + + (let* ([vr-dir "/tmp"] + [target "jcode-local-rejected-draft-escalation.ss"] + [target-path (string-append vr-dir "/" target)] + [bad1 "(import (jerboa prelude))\n(def (main) (displayln \"one\")))\n"] + [bad2 "(import (jerboa prelude))\n(def (main) (displayln \"two\")))\n"] + [good-line "(def (main) (displayln \"two\"))"] + [staged-tool-names '()] + [slurp (lambda (p) + (call-with-input-file p (lambda (i) (get-string-all i))))]) + (safe-delete-test-file! target-path) + (let* ([resp + (lambda (_messages specs step) + (case step + [(0) + (list (make-wtool-call "write" + (list (cons "path" target) + (cons "content" bad1)) #f))] + [(1) + (list (make-wtool-call "write" + (list (cons "path" target) + (cons "content" bad2)) #f))] + [(2) + (let ([names (map tool-spec-name specs)]) + (set! staged-tool-names names) + (list (make-wtool-call "line_edit" + (list (cons "path" target) + (cons "line" 2) + (cons "content" good-line)) #f)))] + [(3) (list (make-wtool-call "verify" '() #f))] + [else (error 'test "unexpected staged repair step")]))] + [result + (verified-run resp "repair a repeatedly rejected local draft" + (list + (cons 'cwd vr-dir) + (cons 'verify-command + (string-append "grep -q '\"two\"' " target)) + (cons 'write-scope (parse-write-scope target)) + (cons 'local-model? #t) + (cons 'max-iterations 8) + (cons 'max-tool-errors 0)))]) + (check! "verified-run: repeated local rejection reaches verify" + result "VERIFIED: exit 0\n") + (check-pred! "verified-run: repeated local rejection focuses staged tools" + staged-tool-names + (lambda (names) + (and (= (length names) 3) + (member "line_edit" names) + (member "replace_def" names) + (member "replace_range" names)))) + (check! "verified-run: local staged line edit promotes repaired file" + (slurp target-path) + (string-append "(import (jerboa prelude))\n" good-line "\n"))) + (safe-delete-test-file! target-path)) + + (let* ([vr-dir "/tmp"] + [target "jcode-auto-repair-mismatched-close.ss"] + [target-path (string-append vr-dir "/" target)] + [bad "(import (jerboa prelude))\n(def (main)\n (let ([x 1])\n (displayln x)])\n"] + [fixed "(import (jerboa prelude))\n(def (main)\n (let ([x 1])\n (displayln x)))\n"] + [tool-results '()] + [slurp (lambda (p) + (call-with-input-file p (lambda (i) (get-string-all i))))]) + (safe-delete-test-file! target-path) + (let* ([resp + (scripted-responder + (list + (list (make-wtool-call "write" + (list (cons "path" target) + (cons "content" bad)) #f)) + (list (make-wtool-call "verify" '() #f))))] + [result + (verified-run resp "repair one deterministic mismatched closer" + (list + (cons 'cwd vr-dir) + (cons 'verify-command + (string-append "grep -q displayln " target)) + (cons 'write-scope (parse-write-scope target)) + (cons 'local-model? #t) + (cons 'max-iterations 5) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results)))))))]) + (check! "verified-run: deterministic mismatched closer repair verifies" + result "VERIFIED: exit 0\n") + (check! "verified-run: deterministic mismatched closer repair writes fixed source" + (slurp target-path) fixed) + (check-pred! "verified-run: deterministic mismatch repair is reported" + (reverse tool-results) + (lambda (xs) + (and (pair? xs) + (str-contains? (car xs) + "after applying a minimal delimiter repair"))))) + (safe-delete-test-file! target-path)) + + (let* ([vr-dir "/tmp"] + [target "jcode-local-repair-cap.txt"] + [target-path (string-append vr-dir "/" target)] + [calls 0] + [caps '()] + [responder + (lambda (_messages _tools _step) + (set! calls (+ calls 1)) + (set! caps (cons (current-max-tokens-cap) caps)) + (case calls + [(1) (list (make-wtool-call "verify" '() #f))] + [(2) (list (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" "repaired\n")) #f))] + [(3) (list (make-wtool-call "verify" '() #f))] + [else (list (make-wtool-call "done" + (list (cons "summary" "fixed")) #f))]))]) + (safe-delete-test-file! target-path) + (verified-run responder "repair with a smaller local completion cap" + (list + (cons 'cwd vr-dir) + (cons 'verify-command (string-append "grep -q repaired " target)) + (cons 'write-scope (parse-write-scope target)) + (cons 'local-model? #t) + (cons 'max-iterations 8))) + (let ([observed (reverse caps)]) + (check! "verified-run: local first-draft completion cap is 8k" + (car observed) 8192) + (check! "verified-run: local post-verifier repair cap is 4k" + (cadr observed) 4096) + (check! "verified-run: local post-edit verify turn stays at 4k" + (caddr observed) 4096)) + (safe-delete-test-file! target-path)) + (let* ([vr-dir "/tmp"] [allowed "jcode-verified-scope-allowed.txt"] [denied "jcode-verified-scope-denied.txt"] @@ -5014,6 +5204,70 @@ (str-contains? s "(displayln \"later\")))"))))) (safe-delete-test-file! target-path)) + (let* ([vr-dir "/tmp"] + [target "jcode-local-required-range-transaction.ss"] + [target-path (string-append vr-dir "/" target)] + [bad-source "(import (jerboa prelude))\n(define (bad)\n (displayln \"open\")\n\n(define (next) 2)\n(define (later)\n (displayln \"later\")))\n"] + [partial-span "(define (bad)\n (displayln \"partial\"))"] + [fixed-source "(import (jerboa prelude))\n(define (bad)\n (displayln \"final\"))\n(define (next) 2)\n(define (later)\n (displayln \"later\"))\n"] + [verify-cmd + (string-append + "if grep -q final " target + "; then true; else printf '%s\\n' 'Exception: invalid context for definition (define (next) 2) at line 5, char 1 of " + target + "' >&2; exit 255; fi")] + [tool-results '()] + [slurp + (lambda (p) + (call-with-input-file p (lambda (i) (get-string-all i))))] + [resp + (scripted-responder + (list + (list (make-wtool-call "verify" '() #f)) + (list + (make-wtool-call "replace_range" + (list (cons "path" target) + (cons "start" 2) + (cons "end" 4) + (cons "content" partial-span)) #f)) + (list + (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" fixed-source)) #f)) + (list (make-wtool-call "verify" '() #f))))]) + (safe-delete-test-file! target-path) + (write-test-output-file target-path + (lambda (o) (display bad-source o)) 'replace) + (let ([result + (verified-run resp + "keep local structural repairs transactional" + (list + (cons 'cwd vr-dir) + (cons 'verify-command verify-cmd) + (cons 'write-scope (parse-write-scope target)) + (cons 'local-model? #t) + (cons 'max-iterations 8) + (cons 'max-tool-errors 3) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results)))))))]) + (check! "verified-run: local transactional range recovery verifies" + result "VERIFIED: exit 0\n") + (check! "verified-run: local transactional recovery writes balanced source" + (slurp target-path) fixed-source)