Reduce verified workflow retry noise
ober
2cbe6f1a85a2693c37c4c4120d38cf22d4f504cd
--- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -2962,6 +2962,22 @@ (> (arg-int args "limit" 0) 0) (> (arg-int args "offset" 0) 0))) +(def (local-pre-edit-external-full-read-block-message cwd args) + (and (current-verified-local-model?) + (= (current-successful-edit-count) 0) + (not (current-rejected-ss-draft)) + (not (current-pending-ss-create-repair)) + (not (explicit-read-range? args)) + (let ((path (arg-path args #f))) + (and path + (outside-scope-path? (scope-path cwd path)) + (read-root-allowed? cwd path) + (string-append + "local-model pre-edit external full-file read refused for " + path + ". The task guidance already provides the required API shape. " + "If one exact external API fact blocks implementation, call read(path,start,end) for a narrow span. Otherwise write the first complete draft now, then call verify."))))) + (def (slice-external-content content args path) (if (explicit-read-range? args) (slice-content content args) @@ -3040,6 +3056,8 @@ (cond ((local-pre-edit-inspection-block-message 'read) => (lambda (msg) (raise-recoverable-tool-error msg 'read))) + ((local-pre-edit-external-full-read-block-message cwd args) + => (lambda (msg) (raise-recoverable-tool-error msg 'read))) ((rejected-draft-hard-recovery-message cwd 'read) => (lambda (msg) (raise-recoverable-tool-error msg 'read))) ((let ((path (arg-path args #f))) --- a/src/jcode/core/workflow-runner.ss +++ b/src/jcode/core/workflow-runner.ss @@ -367,6 +367,11 @@ (and (string? content) (string-prefix? forced-first-edit-retry-marker content))) +(def (suppress-verified-text-history? workflow retry-text-responses? content) + (and retry-text-responses? + (verified-workflow? workflow) + (not (forced-first-edit-retry? content)))) + (def (forced-first-edit-retry-message) (string-append "VERIFIED WORKFLOW REQUIRES A STRUCTURED TOOL CALL NOW. " @@ -540,7 +545,9 @@ (let* ((content (text-response-content response)) (forced-first-edit-retry (forced-first-edit-retry? content))) - (unless forced-first-edit-retry + (unless (or forced-first-edit-retry + (suppress-verified-text-history? + workflow retry-text-responses? content)) (emit! (make-assistant-message content))) (let ((tool-calls (and text-response-tool-calls --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -1573,7 +1573,7 @@ (apply-openai-provider-overrides! body provider) (maybe-apply-logprobs! body provider tools) (apply-prompt-cache-controls! body provider) - (hash-put! body "messages" (map message->json messages)) + (hash-put! body "messages" (map (lambda (m) (openai-message->json provider m)) messages)) (when (and tools (not (null? tools)) (not (model-rejects-tools? (provider-model provider)))) (hash-put! body "tools" tools) @@ -1581,6 +1581,29 @@ (or (current-tool-choice-override) "auto"))) body)) +(def (moonshot-kimi-k3? provider) + (and (equal? (provider-name provider) "openrouter") + (string? (provider-model provider)) + (string-prefix? "moonshotai/kimi-k3" (provider-model provider)))) + +(def (blank-string? s) + (or (not s) + (and (string? s) (string=? s "")))) + +(def (openai-message->json provider msg) + (let ((j (message->json msg))) + ;; Moonshot Kimi K3 rejects assistant history entries whose content is the + ;; empty string. Keep the tool-call history intact and add a minimal text + ;; bridge for that provider only. + (when (and (moonshot-kimi-k3? provider) + (equal? (message-role msg) "assistant") + (blank-string? (hash-get j "content"))) + (hash-put! j "content" + (if (message-tool-calls msg) + "Calling tool." + "Tool call required."))) + j)) + ;; Some models reject the `tools` parameter entirely (e.g. DeepSeek's ;; reasoner returns 400 "does not support Function Calling"). (def (model-rejects-tools? model) @@ -2292,7 +2315,7 @@ (let ((opts (make-hash-table))) (hash-put! opts "include_usage" #t) (hash-put! body "stream_options" opts)) - (hash-put! body "messages" (map message->json messages)) + (hash-put! body "messages" (map (lambda (m) (openai-message->json provider m)) messages)) (when (and tools (not (null? tools)) (not (model-rejects-tools? (provider-model provider)))) (hash-put! body "tools" tools) --- a/test/run.ss +++ b/test/run.ss @@ -3764,9 +3764,51 @@ (cons 'max-iterations 6)))]) (check! "verified-run: remote edit is automatically verified" result "VERIFIED: exit 0\n")) - (check! "verified-run: remote auto verify avoids prose retry" - calls 2) - (safe-delete-test-file! target-path)) + (check! "verified-run: remote auto verify avoids prose retry" + calls 2) + (safe-delete-test-file! target-path)) + + (let* ([w (coding-workflow "true" "/tmp")] + [msgs '()] + [calls 0] + [resp + (lambda (_messages _tools _step) + (set! calls (+ calls 1)) + (case calls + [(1) (make-text-response "Continuing.")] + [(2) (list (make-wtool-call "verify" '() #f))] + [(3) (list (make-wtool-call "done" + '(("summary" . "verified-text-history-ok")) + #f))] + [else (error 'test "unexpected verified text retry step")]))]) + (let ([result + (run-workflow w "go" resp + (list (cons 'max-iterations 6) + (cons 'retry-text-responses? #t) + (cons 'on-message + (lambda (m) (set! msgs (cons m msgs))))))]) + (check! "verified retry-text suppresses prose history result" + result "verified-text-history-ok") + (check! "verified retry-text does not emit prose assistant history" + (let loop ([xs msgs]) + (cond + [(null? xs) #f] + [(and (equal? (message-role (car xs)) "assistant") + (equal? (message-content (car xs)) "Continuing.")) + #t] + [else (loop (cdr xs))])) + #f) + (check-pred! "verified retry-text still emits structured nudge" + msgs + (lambda (xs) + (let loop ([ys xs]) + (cond + [(null? ys) #f] + [(and (equal? (message-role (car ys)) "user") + (str-contains? (message-content (car ys)) + "VERIFIED WORKFLOW REQUIRES A STRUCTURED TOOL CALL NOW")) + #t] + [else (loop (cdr ys))])))))) (let* ([vr-dir "/tmp"] [target "jcode-local-env-first-cap.txt"] @@ -3854,6 +3896,79 @@ (lambda (s) (str-contains? s "FAIL: seeded artifact is broken"))) (safe-delete-test-file! target-path)) + (let* ([vr-dir "/tmp/jcode-verified-local-external-work"] + [target "jcode-verified-local-external-read-budget.txt"] + [target-path (string-append vr-dir "/" target)] + [root "/tmp/jcode-verified-local-external-root"] + [external-path (string-append root "/api.ss")] + [old-roots (getenv "JCODE_READ_ROOTS")] + [tool-results '()]) + (safe-delete-test-file! target-path) + (ensure-test-directory! vr-dir) + (ensure-test-directory! root) + (write-test-output-file external-path + (lambda (o) + (display "line one\nline two\nline three\n" o)) + 'replace) + (let ([resp + (scripted-responder + (list + (list (make-wtool-call "read" + (list (cons "path" external-path)) #f)) + (list (make-wtool-call "read" + (list (cons "path" external-path) + (cons "start" 1) + (cons "end" 1)) #f)) + (list (make-wtool-call "edit" + (list (cons "path" target) + (cons "content" "fixed\n")) #f)) + (list (make-wtool-call "verify" '() #f))))]) + (dynamic-wind + (lambda () + (putenv "JCODE_READ_ROOTS" root)) + (lambda () + (check! "verified-run: local external ranged read still reaches verify" + (verified-run resp "avoid broad external local reads" + (list + (cons 'cwd vr-dir) + (cons 'verify-command + (string-append "grep -q fixed " target)) + (cons 'write-scope (parse-write-scope target)) + (cons 'local-model? #t) + (cons 'max-iterations 8) + (cons 'max-tool-errors 0) + (cons 'on-message + (lambda (m) + (when (equal? (message-role m) "tool") + (set! tool-results + (cons (message-content m) tool-results))))))) + "VERIFIED: exit 0\n") + (check-pred! "verified-run: local external full read is refused before edit" + (reverse tool-results) + (lambda (xs) + (let loop ([ys xs]) + (cond + [(null? ys) #f] + [(and (str-contains? (car ys) "[ToolRecoverableError]") + (str-contains? (car ys) + "external full-file read refused")) + #t] + [else (loop (cdr ys))])))) + (check-pred! "verified-run: local external ranged read is allowed" + (reverse tool-results) + (lambda (xs) + (let loop ([ys xs]) + (cond + [(null? ys) #f] + [(str-contains? (car ys) "line one") #t] + [else (loop (cdr ys))]))))) + (lambda () + (if old-roots + (putenv "JCODE_READ_ROOTS" old-roots) + (putenv "JCODE_READ_ROOTS" ""))))) + (safe-delete-test-file! target-path) + (safe-delete-test-file! external-path)) + (let* ([vr-dir "/tmp"] [target "jcode-local-empty-body-repair.ss"] [target-path (string-append vr-dir "/" target)] @@ -10098,6 +10213,33 @@ (let* ([srv (tcp-listen "127.0.0.1" 0)] [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [tc (restore-tool-call "call_1" "read" "{\"path\":\"Makefile\"}")]) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-sse! srv captured openai-hi-sse) + (let* ([p (make-provider "openrouter" "unit-key" + "moonshotai/kimi-k3" base-url)] + [_ (stream-chat-result + p + (list (make-user-message "inspect") + (make-assistant-message #f) + (make-assistant-message #f (list tc)) + (make-tool-result "call_1" "Makefile contents") + (make-user-message "continue")) + '())] + [req (vector-ref captured 0)]) + (check! "Kimi K3 tool-call history has non-empty assistant content" + (and req (str-contains? req "\"content\":\"Calling tool.\"")) + #t) + (check! "Kimi K3 blank assistant history has non-empty content" + (and req (str-contains? req "\"content\":\"Tool call required.\"")) + #t))) + (lambda () (tcp-close srv)))) + +(let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] [captured (vector #f)]) (dynamic-wind (lambda () (void))