Make verified syntax rejections recoverable
ober
bdc64a9678e90d1d035e2c35bd6f7afe188c3f91
--- a/src/jcode/core/errors.ss +++ b/src/jcode/core/errors.ss @@ -29,6 +29,9 @@ raise-tool-execution-error &tool-resolution-error tool-resolution-error? tool-resolution-error-tool raise-tool-resolution-error + &recoverable-tool-error recoverable-tool-error? + recoverable-tool-error-tool + raise-recoverable-tool-error &workflow-cancelled workflow-cancelled-error? workflow-cancelled-error-iteration workflow-cancelled-error-completed raise-workflow-cancelled @@ -135,6 +138,20 @@ (make-tool-resolution-error tool-name) (make-message-condition msg))))) +;; RecoverableToolError — a tool rejected an unsafe/invalid operation but +;; produced concrete next-step guidance. The runner emits it as a tool result +;; and does not consume the hard tool-error budget, keeping repair loops alive. +(define-condition-type &recoverable-tool-error &error + make-recoverable-tool-error recoverable-tool-error? + (tool recoverable-tool-error-tool)) + +(def (raise-recoverable-tool-error msg . opt) + (let ((tool-name (and (pair? opt) (car opt)))) + (raise + (condition + (make-recoverable-tool-error tool-name) + (make-message-condition msg))))) + ;; WorkflowCancelledError — cancel-event set before completion. (define-condition-type &workflow-cancelled &forge-error make-workflow-cancelled workflow-cancelled-error? --- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -27,6 +27,7 @@ :jcode/core/verified :jcode/core/best-of-k :jcode/core/workflow-runner + :jcode/core/errors :jcode/tool/registry :jcode/proxy/server) @@ -165,9 +166,11 @@ ;; runner wants (messages tool-specs step-index). Ignore the step index and pass ;; no request-level sampling — the provider applies its own per-model policy. (def (provider-responder provider) - (let ((backend (make-provider-backend provider))) - (lambda (messages tool-specs _step) - (backend messages tool-specs #f)))) + (if (procedure? provider) + provider + (let ((backend (make-provider-backend provider))) + (lambda (messages tool-specs _step) + (backend messages tool-specs #f))))) ;; ── verify oracle ────────────────────────────────────────────────────── (def (tail-lines s n) @@ -780,6 +783,9 @@ (def current-rejected-draft-path-reject-count (make-parameter 0)) +(def current-rejected-draft-needs-inspection + (make-parameter #f)) + (def current-existing-ss-rewrite-reject-path (make-parameter #f)) @@ -861,7 +867,8 @@ (current-rejected-draft-fingerprint #f) (current-rejected-draft-repeat-count 0) (current-rejected-draft-path #f) - (current-rejected-draft-path-reject-count 0)) + (current-rejected-draft-path-reject-count 0) + (current-rejected-draft-needs-inspection #f)) (def (reset-existing-ss-rewrite-state!) (current-existing-ss-rewrite-reject-path #f) @@ -939,14 +946,16 @@ (let* ((fp (rejected-draft-fingerprint path content)) (same? (equal? fp (current-rejected-draft-fingerprint))) (old-path (current-rejected-draft-path)) - (same-path? (and old-path (string=? old-path path)))) + (same-path? (and old-path (string=? old-path path))) + (path-reject-count (if same-path? + (+ (current-rejected-draft-path-reject-count) 1) + 1))) (current-rejected-ss-draft (cons path content)) (current-rejected-draft-fingerprint fp) (current-rejected-draft-path path) - (current-rejected-draft-path-reject-count - (if same-path? - (+ (current-rejected-draft-path-reject-count) 1) - 1)) + (current-rejected-draft-path-reject-count path-reject-count) + (current-rejected-draft-needs-inspection + (and same-path? (>= path-reject-count 2))) (if same? (let ((n (+ (current-rejected-draft-repeat-count) 1))) (current-rejected-draft-repeat-count n) @@ -981,6 +990,21 @@ "\", start=<near reported line>, end=<near reported line>) on the latest rejected draft, " "then send one complete corrected file body with edit/write."))) +(def (rejected-draft-inspection-required-message path) + (and (current-rejected-draft-needs-inspection) + (let ((draft (current-rejected-ss-draft))) + (and (pair? draft) + (string=? (car draft) path) + (string-append + "Another broad full-file write for " + path + " is blocked until the rejected draft is inspected. " + "Next call must be read(path=\"" + path + "\", start=1, end=<small line window>) or balance(path=\"" + path + "\"). Then send one corrected complete file body with edit/write and call verify."))))) + (def (tool-label who) (cond ((symbol? who) (symbol->string who)) @@ -1323,6 +1347,7 @@ (def (note-rejected-draft-inspection! path content) (let ((n (+ (current-rejected-draft-inspections) 1))) (current-rejected-draft-inspections n) + (current-rejected-draft-needs-inspection #f) (and (> n rejected-draft-inspection-limit) (rejected-draft-limit-message path content)))) @@ -2289,9 +2314,9 @@ (if repair (required-repair-balance-message cwd repair) "balance: missing path"))) + ((rejected-draft-balance-message cwd path) => (lambda (msg) msg)) ((rejected-draft-hard-recovery-message cwd 'balance) => (lambda (msg) msg)) ((note-inspection-after-edit! 'balance) => (lambda (msg) (error 'balance msg))) - ((rejected-draft-balance-message cwd path) => (lambda (msg) msg)) ((current-required-range-repair) => (lambda (repair) (required-repair-balance-message cwd repair))) ((pending-ss-create-repair-message cwd) => (lambda (msg) msg)) @@ -2906,12 +2931,13 @@ (when record-pending? (current-pending-ss-create-repair path)) (record-rejected-ss-draft! path content)) - (error 'edit - (string-append msg - "\n" - (ss-repair-instruction path) - (or (rejected-draft-repeat-note path) "") - (or (rejected-draft-blind-loop-note path) "")))))) + (raise-recoverable-tool-error + (string-append msg + "\n" + (ss-repair-instruction path) + (or (rejected-draft-repeat-note path) "") + (or (rejected-draft-blind-loop-note path) "")) + 'edit)))) (def (guard-jerboa-syntax-existing-file-edit! cwd path content) (let ((msg (jerboa-syntax-guard-message path content))) @@ -2919,29 +2945,31 @@ (if (scaffold-created-ss? cwd path) (begin (record-rejected-ss-draft! path content) - (error 'edit - (string-append - msg - "\nThe full-file replacement was not written; the scaffold on disk is unchanged. " - "This .ss file was created by create_verified_jerboa_script during this workflow, " - "so you may retry edit/write with complete corrected contents, or inspect the rejected draft with read(path=\"" - path - "\") or balance(path=\"" - path - "\"). Call verify after the replacement is written." - (or (rejected-draft-repeat-note path) "") - (or (rejected-draft-blind-loop-note path) "")))) + (raise-recoverable-tool-error + (string-append + msg + "\nThe full-file replacement was not written; the scaffold on disk is unchanged. " + "This .ss file was created by create_verified_jerboa_script during this workflow, " + "so you may retry edit/write with complete corrected contents, or inspect the rejected draft with read(path=\"" + path + "\") or balance(path=\"" + path + "\"). Call verify after the replacement is written." + (or (rejected-draft-repeat-note path) "") + (or (rejected-draft-blind-loop-note path) "")) + 'edit)) (begin (record-existing-ss-rewrite-rejection! cwd path content) (reset-rejected-draft-state!) - (error 'edit - (string-append - msg - "\nThe full-file rewrite was not written; the on-disk file is unchanged. " - "Do not inspect or repair this rejected full-file draft. Next tool call should be balance(path=\"" - path - "\") or verify(), then use line_edit, replace_range, or replace_def for a local repair.\n" - (existing-ss-rewrite-lock-message cwd path)))))))) + (raise-recoverable-tool-error + (string-append + msg + "\nThe full-file rewrite was not written; the on-disk file is unchanged. " + "Do not inspect or repair this rejected full-file draft. Next tool call should be balance(path=\"" + path + "\") or verify(), then use line_edit, replace_range, or replace_def for a local repair.\n" + (existing-ss-rewrite-lock-message cwd path)) + 'edit)))))) (def (replace-range-syntax-rejection-message path start-line end-line msg) (string-append @@ -3221,6 +3249,11 @@ (not (looks-like-complete-file? content))) (reject-incomplete-ss-create path content "full-write snippets")) + ((and (not (file-exists? p)) + (source-ss-path? path) + (rejected-draft-inspection-required-message path)) + => (lambda (msg) + (raise-recoverable-tool-error msg 'edit))) ((and (file-exists? p) (existing-ss-rewrite-locked? cwd path) (not required-full-rewrite?)) --- a/src/jcode/core/workflow-runner.ss +++ b/src/jcode/core/workflow-runner.ss @@ -61,10 +61,12 @@ ;; Run one tool callable, classifying the outcome: ;; (ok . value) — succeeded ;; (resolution . text) — ToolResolutionError (privileged: no error budget) +;; (recoverable . text) — recoverable guidance (privileged: no error budget) ;; (error . text) — any other exception (counts against error budget) ;; The callable receives the args assoc as its single argument. (def (run-one-tool fn args) (guard (e [(tool-resolution-error? e) (cons 'resolution (condition->string e))] + [(recoverable-tool-error? e) (cons 'recoverable (condition->string e))] [#t (cons 'error (condition->string e))]) (cons 'ok (fn args)))) @@ -181,7 +183,14 @@ (case (car outcome) ((resolution) ;; privileged: emit result, no error-budget hit - (emit! (make-tool-result tc-id (string-append "[ToolResolutionError] " (cdr outcome)))) + (emit! (make-tool-result tc-id (string-append "[ToolResolutionError] " (cdr outcome)))) + (loop (cdr tcs) (cdr tds) had-error last-error + (if terminal? 'errored terminal))) + ((recoverable) + ;; Tool produced concrete repair guidance for a refused operation. + ;; Do not record the step as complete, but keep the hard tool-error + ;; budget for genuine failures after the model follows guidance. + (emit! (make-tool-result tc-id (string-append "[ToolRecoverableError] " (cdr outcome)))) (loop (cdr tcs) (cdr tds) had-error last-error (if terminal? 'errored terminal))) ((error) --- a/test/run.ss +++ b/test/run.ss @@ -1467,6 +1467,39 @@ (check! "resolution-error privileged → eventual success" result "RESOLVED-OK") (check! "resolution retried exactly twice" calls 2)) +;; Recoverable tool guidance is also privileged: the tool did not complete, but +;; the model gets concrete next-step feedback without spending hard error budget. +(let* ([calls 0] + [w (make-workflow "rec" "d" + (list (make-tool-def (make-tool-spec "repair" "r" '()) + (lambda (a) + (set! calls (+ calls 1)) + (if (< calls 3) + (raise-recoverable-tool-error "inspect rejected draft" "repair") + "RECOVERED-OK")) + '())) + '() "repair" "p")] + [tool-results '()] + [resp (scripted-responder + (list (list (make-wtool-call "repair" '() #f)) + (list (make-wtool-call "repair" '() #f)) + (list (make-wtool-call "repair" '() #f))))] + [result (run-workflow w "go" resp + (list (cons 'max-iterations 6) + (cons 'max-tool-errors 0) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results)))))))]) + (check! "recoverable tool guidance does not exhaust error budget" + result "RECOVERED-OK") + (check-pred! "recoverable tool guidance is emitted distinctly" + (reverse tool-results) + (lambda (xs) + (and (pair? xs) + (str-contains? (car xs) "[ToolRecoverableError] inspect rejected draft"))))) + ;; Unknown workflow tools are surfaced as privileged ToolResolutionError ;; results so local models can recover instead of crashing the runner. (let* ([w (mk-research-wf)] @@ -2052,13 +2085,75 @@ (reverse tool-results))) (check! "verified-run: gate held — done after passing verify, not the buggy one" result "verified-and-done") - (check! "verified-run: do-edit wrote the repaired bytes to the real file" - (slurp vr-path) "correct")) - (safe-delete-test-file! vr-path)) + (check! "verified-run: do-edit wrote the repaired bytes to the real file" + (slurp vr-path) "correct")) + (safe-delete-test-file! vr-path)) -(let* ([vr-dir "/tmp"] - [allowed "jcode-verified-scope-allowed.txt"] - [denied "jcode-verified-scope-denied.txt"] + (let* ([vr-dir "/tmp"] + [target "jcode-verified-run-fake-provider-rejected.ss"] + [target-path (string-append vr-dir "/" target)] + [bad1 "(import (jerboa prelude))\n(define (main)\n (displayln \"bad1\")))\n"] + [bad2 "(import (jerboa prelude))\n(define (main)\n (displayln \"bad2\")))\n"] + [good "(import (jerboa prelude))\n(define (main)\n (displayln \"fixed\"))\n"] + [tool-results '()] + [slurp (lambda (p) (call-with-input-file p (lambda (i) (get-string-all i))))]) + (safe-delete-test-file! target-path) + (let* ([resp (scripted-responder + (list + (list (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" bad1)) + #f)) + (list (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" bad2)) + #f)) + (list (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" good)) + #f)) + (list (make-wtool-call "balance" + (list (cons "path" target)) + #f)) + (list (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" good)) + #f)) + (list (make-wtool-call "verify" '() #f))))] + [result (verified-run resp "repair rejected .ss create" + (list (cons 'cwd vr-dir) + (cons 'verify-command + (string-append "grep -q fixed " target)) + (cons 'write-scope (parse-write-scope target)) + (cons 'max-iterations 10) + (cons 'max-tool-errors 0) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results)))))))]) + (check! "verified-run: fake provider recovers rejected creates with zero hard errors" + result "VERIFIED: exit 0\n") + (check! "verified-run: fake provider wrote repaired file" + (slurp target-path) good) + (check-pred! "verified-run: repeated broad create forces inspection first" + (reverse tool-results) + (lambda (xs) + (let loop ([ys xs] [saw-recoverable #f] [saw-block #f]) + (cond + [(null? ys) (and saw-recoverable saw-block)] + [else + (loop (cdr ys) + (or saw-recoverable + (str-contains? (car ys) "[ToolRecoverableError]")) + (or saw-block + (str-contains? (car ys) + "blocked until the rejected draft is inspected")))]))))) + (safe-delete-test-file! target-path)) + + (let* ([vr-dir "/tmp"] + [allowed "jcode-verified-scope-allowed.txt"] + [denied "jcode-verified-scope-denied.txt"] [allowed-path (string-append vr-dir "/" allowed)] [denied-path (string-append vr-dir "/" denied)] [slurp (lambda (p) (call-with-input-file p (lambda (i) (get-string-all i))))]) @@ -4196,6 +4291,11 @@ #f)) (list (make-wtool-call + "balance" + (list (cons "path" target)) + #f)) + (list + (make-wtool-call "edit" (list (cons "path" target) (cons "content" good)) @@ -4322,6 +4422,11 @@ #f)) (list (make-wtool-call + "balance" + (list (cons "path" target)) + #f)) + (list + (make-wtool-call "edit" (list (cons "path" target) (cons "content" good)) @@ -4330,7 +4435,7 @@ (list (make-wtool-call "done" '(("summary" . "rejected-draft-repeat-ok")) #f))))] [result (parameterize ((current-write-scope scope)) (run-workflow wf "repeat identical rejected draft" resp - (list (cons 'max-iterations 12) + (list (cons 'max-iterations 14) (cons 'max-tool-errors 4) (cons 'on-message (lambda (m) @@ -4957,10 +5062,11 @@ (let loop ([ys xs] [saw-guard #f] [saw-lock #f]) (cond [(null? ys) (and saw-guard saw-lock)] - [else + [else (loop (cdr ys) (or saw-guard - (and (str-contains? (car ys) "[ToolError]") + (and (or (str-contains? (car ys) "[ToolError]") + (str-contains? (car ys) "[ToolRecoverableError]")) (str-contains? (car ys) "full-file rewrite was not written"))) (or saw-lock (and (str-contains? (car ys) "Full-file rewrite is locked")