Limit pre-edit shell alias probing
ober
fa52b33b8d7075c74493bcbf7f3ab001db606c8b
--- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -633,6 +633,16 @@ name ". Stop routing to MCP/API discovery. The next useful tool must be read/list on the current repo if a local file is still unknown, or edit/write with the first complete draft. After a successful edit and verify failure, use targeted MCP only for concrete syntax/API questions."))))) +(def (pre-edit-run-alias-block-message tool-name) + (and (= (current-successful-edit-count) 0) + (let ((next (+ (current-pre-edit-run-alias-count) 1))) + (current-pre-edit-run-alias-count next) + (and (> next pre-edit-run-alias-limit) + (string-append + "pre-edit run/bash/shell inspection limit reached while calling " + tool-name + ". Stop using shell-shaped exploration. The next useful tool must be read/list on a specific current-repo path if one file is still unknown, or edit/write with the first complete draft, then verify(). Do not probe environment paths, HOME, PATH, or external installs."))))) + (def (post-failed-verify-mcp-block-message name) (and (current-after-failed-verify?) (not (verified-targeted-after-failure-mcp-tool? name)) @@ -847,6 +857,9 @@ (def current-pre-edit-mcp-count (make-parameter 0)) +(def current-pre-edit-run-alias-count + (make-parameter 0)) + (def current-last-verify-detail (make-parameter #f)) @@ -862,6 +875,8 @@ (def pre-edit-mcp-limit 2) +(def pre-edit-run-alias-limit 4) + (def rejected-draft-inspection-limit 2) (def rejected-draft-repeat-threshold 1) @@ -1059,6 +1074,7 @@ (reset-existing-ss-rewrite-state!) (reset-path-only-run-state!) (reset-failed-verify-inspections!) + (current-pre-edit-run-alias-count 0) (current-successful-edit-count (+ (current-successful-edit-count) 1)) (current-edited-since-verify? #t) (current-inspections-after-edit 0)) @@ -2648,6 +2664,8 @@ ((cat-heredoc-write cmd cwd) => (lambda (result) result)) (else (cond + ((pre-edit-run-alias-block-message "run") + => (lambda (msg) (raise-recoverable-tool-error msg 'run))) ((note-inspection-after-failed-verify! 'run) => (lambda (msg) (error 'run msg))) ((and (string? cmd) (not (null? words)) @@ -3537,6 +3555,7 @@ (def (coding-workflow verify-cmd cwd . opt) (current-successful-edit-count 0) (current-pre-edit-mcp-count 0) + (current-pre-edit-run-alias-count 0) (let* ((o (if (pair? opt) (car opt) '())) (scope-pair (assoc 'write-scope o)) (scope (if scope-pair @@ -3771,7 +3790,11 @@ "Compact verified mode: inspect only the few files needed to remove uncertainty, then write a complete first version and call verify. Before the first write, prefer read/list on the current repo or JCODE_READ_ROOTS over broad MCP/API discovery. After verify fails, repair the concrete verifier error with the smallest edit and verify again.\n" "") (if run-aliases? - "run/bash/shell are narrow inspection aliases only; use verify for the configured build/test command.\n" + (string-append + "run/bash/shell are narrow inspection aliases only; use verify for the configured build/test command. " + "Before the first edit, run/bash/shell inspection has a hard budget of " + (number->string pre-edit-run-alias-limit) + " calls; then write the first complete draft and verify.\n") "run/bash/shell are not available in this workflow. Use verify for the configured build/test command.\n") "For a new Jerboa .ss script, create_verified_jerboa_script is optional. Use it only when one generic kind clearly matches: minimal-pass for trivial executable scripts, cli-two-args for simple numeric two-argument CLIs, and vector-grid for 2-D vector/grid examples. For nontrivial algorithms, write a small complete first version that parses and runs, verify it, then expand one function or data transformation at a time using verifier output.\n" (external-tools-instruction external-tool-defs) @@ -3840,6 +3863,7 @@ (current-inspections-after-edit 0) (current-successful-edit-count 0) (current-pre-edit-mcp-count 0) + (current-pre-edit-run-alias-count 0) (current-required-range-repair #f)) (if (> k 1) (run-best-of-k wf task (lambda () (provider-responder provider)) k ropt) --- a/test/run.ss +++ b/test/run.ss @@ -2350,6 +2350,54 @@ (str-contains? s "auto-converted to list") (str-contains? s "Do not retry run/bash/shell"))))) +(let* ([vr-dir "/tmp"] + [fixture "jcode-verified-run-budget-fixture.txt"] + [fixture-path (string-append vr-dir "/" fixture)] + [target "jcode-verified-run-budget-out.txt"] + [target-path (string-append vr-dir "/" target)] + [tool-results '()] + [wf (coding-workflow (string-append "grep -q done " target) vr-dir + (list (cons 'run-aliases? #t)))] + [resp (scripted-responder + (list + (list (make-wtool-call "run" '(("command" . "ls")) #f)) + (list (make-wtool-call "run" (list (cons "command" (string-append "cat " fixture))) #f)) + (list (make-wtool-call "run" (list (cons "command" (string-append "wc -l " fixture))) #f)) + (list (make-wtool-call "run" (list (cons "command" (string-append "head -1 " fixture))) #f)) + (list (make-wtool-call "run" '(("command" . "echo HOME=$HOME")) #f)) + (list (make-wtool-call "edit" (list (cons "path" target) + (cons "content" "done\n")) #f)) + (list (make-wtool-call "verify" '() #f)) + (list (make-wtool-call "done" '(("summary" . "run-budget-recovered")) #f))))]) + (safe-delete-test-file! fixture-path) + (safe-delete-test-file! target-path) + (write-test-output-file fixture-path + (lambda (o) (display "fixture\n" o)) + 'replace) + (let ([result (run-workflow wf "stop shell probing and write" resp + (list (cons 'max-iterations 12) + (cons 'max-tool-errors 0) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results)))))))]) + (check! "verified-run: pre-edit run alias budget recovers with zero hard errors" + result "run-budget-recovered") + (check-pred! "verified-run: pre-edit run alias budget gives recoverable guidance" + (reverse tool-results) + (lambda (xs) + (let loop ([ys xs]) + (cond + [(null? ys) #f] + [(and (str-contains? (car ys) "[ToolRecoverableError]") + (str-contains? (car ys) "pre-edit run/bash/shell inspection limit reached") + (str-contains? (car ys) "first complete draft")) + #t] + [else (loop (cdr ys))]))))) + (safe-delete-test-file! fixture-path) + (safe-delete-test-file! target-path)) + (let ([old-read-roots (getenv "JCODE_READ_ROOTS")]) (dynamic-wind (lambda () (putenv "JCODE_READ_ROOTS" ""))