Force remote verified runs to first edit
ober
4c556c4e97d54f02989f7665587719d40f630d4a
--- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -262,16 +262,19 @@ #f) ;; Keep cheap directory navigation separate from source inspection, ;; so locating an approved dependency does not consume every read. - ((and (current-verified-local-model?) - (= (current-successful-edit-count) 0) + ((and (= (current-successful-edit-count) 0) (not (current-rejected-ss-draft)) (not (current-pending-ss-create-repair)) (or (and (verified-navigation-tool-spec? spec) (>= (current-pre-edit-navigation-count) - local-pre-edit-navigation-limit)) + (if (current-verified-local-model?) + local-pre-edit-navigation-limit + remote-pre-edit-navigation-limit))) (and (not (verified-navigation-tool-spec? spec)) (>= (current-pre-edit-inspection-count) - local-pre-edit-inspection-limit))) + (if (current-verified-local-model?) + local-pre-edit-inspection-limit + remote-pre-edit-inspection-limit)))) (verified-inspection-tool-spec? spec)) #f) ;; A local model that has consumed its focused post-verifier reads @@ -335,25 +338,37 @@ ((string=? (tool-spec-name (car xs)) name) #t) (else (loop (cdr xs)))))) -(def (local-force-first-edit-tool-choice specs) - (and (current-verified-local-model?) - (= (current-successful-edit-count) 0) +(def remote-pre-edit-inspection-limit 12) +(def remote-pre-edit-navigation-limit 10) +(def remote-force-first-edit-inspection-count 10) + +(def (pre-edit-inspection-total) + (+ (current-pre-edit-inspection-count) + (current-pre-edit-navigation-count) + (current-pre-edit-mcp-count) + (current-pre-edit-run-alias-count))) + +(def (force-first-edit-tool-choice specs) + (and (= (current-successful-edit-count) 0) (not (current-edited-since-verify?)) (not (current-after-failed-verify?)) (not (current-rejected-ss-draft)) (not (current-pending-ss-create-repair)) - (>= (current-pre-edit-inspection-count) 2) + (if (current-verified-local-model?) + (>= (current-pre-edit-inspection-count) 2) + (>= (pre-edit-inspection-total) + remote-force-first-edit-inspection-count)) (tool-specs-include? specs "edit") (openai-function-tool-choice "edit"))) -(def (local-forced-first-edit-specs specs forced-choice) +(def (forced-first-edit-specs specs forced-choice) (if forced-choice (filter (lambda (spec) (member (tool-spec-name spec) '("edit" "write"))) specs) specs)) -(def (local-forced-first-edit-response response forced-choice) +(def (forced-first-edit-response response forced-choice) (if (and forced-choice (pair? response) (let loop ((xs response)) @@ -365,7 +380,7 @@ #t) (else (loop (cdr xs)))))) (make-text-response - "Forced first-edit turn rejected: the model called a hidden inspection or non-edit tool. Do not call read, list, verify, or any other tool on this turn. Call exactly edit(path=\"tetris.ss\", content=<complete minimal acceptance-complete file body>) or write(path=\"tetris.ss\", content=<complete minimal acceptance-complete file body>) now.") + "Forced first-edit turn rejected: the model called a hidden inspection or non-edit tool. Do not call read, list, balance, verify, or any other tool on this turn. Call exactly edit(path=<scoped target file>, content=<complete minimal acceptance-complete file body>) or write(path=<scoped target file>, content=<complete minimal acceptance-complete file body>) now.") response)) (def (provider-responder provider) @@ -386,7 +401,14 @@ default-local-verified-repair-max-completion-tokens (current-max-tokens-cap)))) (if (procedure? provider) - (backend messages (verified-provider-tool-specs tool-specs) step) + (let* ((specs (verified-provider-tool-specs tool-specs)) + (forced-choice (and (not (current-verified-local-model?)) + (force-first-edit-tool-choice specs)))) + (forced-first-edit-response + (backend messages + (forced-first-edit-specs specs forced-choice) + step) + forced-choice)) (let* ((sticky-rejected-draft? (and (current-verified-local-model?) (current-rejected-ss-draft) @@ -407,12 +429,12 @@ ((make-provider-backend (get-expert-provider) chat-direct-via-stream) messages specs #f)) - (let ((forced-choice (local-force-first-edit-tool-choice specs))) + (let ((forced-choice (force-first-edit-tool-choice specs))) (parameterize ((current-tool-choice-override forced-choice)) - (local-forced-first-edit-response + (forced-first-edit-response (backend messages - (local-forced-first-edit-specs specs forced-choice) + (forced-first-edit-specs specs forced-choice) #f) forced-choice)))))))))) @@ -1509,8 +1531,7 @@ ". Stop using shell-shaped exploration. The next useful tool must be read/list on a specific current-repo path if one file is still unknown, or edit/write with the first complete draft, then verify(). Do not probe environment paths, HOME, PATH, or external installs."))))) (def (local-pre-edit-inspection-block-message tool-name) - (and (current-verified-local-model?) - (= (current-successful-edit-count) 0) + (and (= (current-successful-edit-count) 0) (not (current-rejected-ss-draft)) (not (current-pending-ss-create-repair)) (let* ((navigation? (member tool-name '(list ls))) @@ -1518,17 +1539,28 @@ (current-pre-edit-navigation-count) (current-pre-edit-inspection-count)) 1)) - (limit (if navigation? - local-pre-edit-navigation-limit - local-pre-edit-inspection-limit))) + (limit (cond + ((and navigation? (current-verified-local-model?)) + local-pre-edit-navigation-limit) + (navigation? remote-pre-edit-navigation-limit) + ((current-verified-local-model?) + local-pre-edit-inspection-limit) + (else remote-pre-edit-inspection-limit)))) (if navigation? (current-pre-edit-navigation-count next) (current-pre-edit-inspection-count next)) - (and (> next limit) + (and (current-verified-local-model?) + (> next limit) (string-append - (if navigation? - "local-model pre-edit navigation limit reached while calling " - "local-model pre-edit inspection limit reached while calling ") + (cond + ((and navigation? (current-verified-local-model?)) + "local-model pre-edit navigation limit reached while calling ") + (navigation? + "pre-edit navigation limit reached while calling ") + ((current-verified-local-model?) + "local-model pre-edit inspection limit reached while calling ") + (else + "pre-edit inspection limit reached while calling ")) (tool-label tool-name) ". The repository and task already provide enough context. Stop inspecting external files and write the first complete draft now; then call verify. Inspection tools are hidden until the first edit."))))) --- a/test/run.ss +++ b/test/run.ss @@ -5721,6 +5721,77 @@ (safe-delete-test-file! target-path)) (let* ([vr-dir "/tmp"] + [target "jcode-verified-remote-force-first-edit.txt"] + [target-path (string-append vr-dir "/" target)] + [readme "jcode-verified-remote-force-first-edit-readme.txt"] + [readme-path (string-append vr-dir "/" readme)] + [forced-spec-names '()] + [tool-results '()] + [calls 0] + [read-call + (lambda (line) + (list (make-wtool-call "read" + (list (cons "path" readme) + (cons "start" line) + (cons "end" line)) #f)))] + [responder + (lambda (_messages specs _step) + (set! calls (+ calls 1)) + (when (= calls 11) + (set! forced-spec-names (map tool-spec-name specs))) + (cond + [(<= calls 10) (read-call calls)] + [(= calls 11) + ;; Simulate a remote model ignoring the narrowed schema. + (read-call calls)] + [(= calls 12) + (list (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" "fixed\n")) #f))] + [(= calls 13) + (list (make-wtool-call "verify" '() #f))] + [else (error 'test "unexpected remote first-edit force step")]))]) + (safe-delete-test-file! target-path) + (write-test-output-file readme-path + (lambda (o) + (let loop ([n 1]) + (when (<= n 20) + (display (string-append "context-" (number->string n) "\n") o) + (loop (+ n 1))))) + 'replace) + (let ([result + (verified-run responder "force remote first edit after inspection loop" + (list + (cons 'cwd vr-dir) + (cons 'verify-command (string-append "grep -q fixed " target)) + (cons 'write-scope (parse-write-scope target)) + (cons 'max-iterations 20) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results)))))))]) + (check! "verified-run: remote first-edit force reaches verify" + result "VERIFIED: exit 0\n") + (check-pred! "verified-run: remote first-edit force hides inspection" + forced-spec-names + (lambda (names) + (and (member "edit" names) + (member "write" names) + (not (member "read" names)) + (not (member "list" names)) + (not (member "balance" names))))) + (check! "verified-run: remote first-edit force wrote file" + (call-with-input-file target-path (lambda (p) (get-string-all p))) + "fixed\n") + (check! "verified-run: hidden remote read was not executed" + (length (filter (lambda (s) (str-contains? s "context-")) + (reverse tool-results))) + 10)) + (safe-delete-test-file! target-path) + (safe-delete-test-file! readme-path)) + + (let* ([vr-dir "/tmp"] [target "jcode-verified-oldstr-miss.txt"] [target-path (string-append vr-dir "/" target)] [tool-results '()]