Bound forced tool reasoning streams
ober
7f4681ad82fc0947fb8451001ed96e66adc8c4fc
--- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -1023,11 +1023,17 @@ (def forced-tool-choice-retry-marker "JCODE_INTERNAL_FORCE_FIRST_EDIT_RETRY") -(def (forced-tool-choice-retry-content expected actual) +(def *forced-tool-no-call-default-char-limit* 4096) + +(def (forced-tool-no-call-char-limit) + (or (positive-int-value (getenv "JCODE_FORCED_TOOL_NO_CALL_CHAR_LIMIT")) + *forced-tool-no-call-default-char-limit*)) + +(def (forced-tool-choice-retry-content expected detail) (string-append forced-tool-choice-retry-marker - " provider streamed hidden tool " - actual + " " + detail " while this turn required " expected ".")) @@ -2390,7 +2396,9 @@ (saw-done-box (box #f)) (forced-tool-name (openai-tool-choice-name (current-tool-choice-override))) - (forced-tool-mismatch-box (box #f))) + (forced-tool-mismatch-box (box #f)) + (forced-tool-pre-call-chars-box (box 0)) + (forced-tool-pre-call-char-limit (forced-tool-no-call-char-limit))) (let* ((body-json (json-object->string body)) (dummy (begin (log-info logger "stream-request" @@ -2444,6 +2452,23 @@ (let ((content (hash-get delta "content"))) (when (and content (string? content) (> (string-length content) 0)) (put-string text-acc content) + (when (and forced-tool-name + (null? (hash-keys tc-table))) + (set-box! forced-tool-pre-call-chars-box + (+ (unbox forced-tool-pre-call-chars-box) + (string-length content))) + (when (> (unbox forced-tool-pre-call-chars-box) + forced-tool-pre-call-char-limit) + (set-box! forced-tool-mismatch-box + (cons forced-tool-name + (format "provider streamed ~a text/reasoning characters without calling a tool" + (unbox forced-tool-pre-call-chars-box)))) + (log-warn logger "forced-tool-stream-no-call" + `((expected . ,forced-tool-name) + (chars . ,(unbox forced-tool-pre-call-chars-box)) + (limit . ,forced-tool-pre-call-char-limit))) + (error 'openai-stream-chat + "provider streamed too much text during forced tool turn"))) (token-cb content))) ;; Reasoning token (mlx-omni `reasoning`, R1 ;; `reasoning_content`). Stream to UI so the user @@ -2453,6 +2478,23 @@ (hash-get delta "reasoning_content")))) (when (and r (string? r) (> (string-length r) 0)) (put-string reasoning-acc r) + (when (and forced-tool-name + (null? (hash-keys tc-table))) + (set-box! forced-tool-pre-call-chars-box + (+ (unbox forced-tool-pre-call-chars-box) + (string-length r))) + (when (> (unbox forced-tool-pre-call-chars-box) + forced-tool-pre-call-char-limit) + (set-box! forced-tool-mismatch-box + (cons forced-tool-name + (format "provider streamed ~a text/reasoning characters without calling a tool" + (unbox forced-tool-pre-call-chars-box)))) + (log-warn logger "forced-tool-stream-no-call" + `((expected . ,forced-tool-name) + (chars . ,(unbox forced-tool-pre-call-chars-box)) + (limit . ,forced-tool-pre-call-char-limit))) + (error 'openai-stream-chat + "provider streamed too much reasoning during forced tool turn"))) (token-cb r))) ;; Tool call fragments (let ((tcs (hash-get delta "tool_calls"))) @@ -2475,7 +2517,10 @@ (string? name) (not (string=? name forced-tool-name))) (set-box! forced-tool-mismatch-box - (cons forced-tool-name name)) + (cons forced-tool-name + (string-append + "provider streamed hidden tool " + name))) (log-warn logger "forced-tool-stream-mismatch" `((expected . ,forced-tool-name) (actual . ,name))) --- a/test/run.ss +++ b/test/run.ss @@ -13130,6 +13130,54 @@ (tcp-close srv)))) (let* ([sse-body + (string-append + "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"drafting code in reasoning instead of calling edit\"},\"finish_reason\":null}]}\n\n" + "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\" and still no tool call\"},\"finish_reason\":null}]}\n\n")] + [srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [edit-tool (uuid-recovery-tool "edit" (args "path" (args "type" "string")) '("path"))] + [old-max-tokens (getenv "JCODE_MAX_TOKENS")] + [old-limit (getenv "JCODE_FORCED_TOOL_NO_CALL_CHAR_LIMIT")]) + (dynamic-wind + (lambda () + (putenv "JCODE_MAX_TOKENS" "") + (putenv "JCODE_FORCED_TOOL_NO_CALL_CHAR_LIMIT" "32")) + (lambda () + (serve-one-captured-sse! srv captured sse-body) + (let* ([p (make-provider "mlx" "" "deepseek-v4-pro" base-url)] + [result (call-with-values + (lambda () + (parameterize ([current-tool-choice-override + (openai-function-tool-choice "edit")]) + (provider-stream-chat + p + (list (make-user-message "hi")) + (list edit-tool) + (lambda (_token) #f)))) + (lambda (content tcs usage) (list content tcs usage)))] + [req (vector-ref captured 0)]) + (check-pred! "mlx forced streaming reasoning-only returns retry marker" + (car result) + (lambda (s) + (and (string? s) + (str-contains? s + "JCODE_INTERNAL_FORCE_FIRST_EDIT_RETRY") + (str-contains? s + "without calling a tool") + (str-contains? s "required edit")))) + (check! "mlx forced streaming reasoning-only has no tool call" + (list-ref result 1) '()) + (check! "mlx forced streaming reasoning-only request required edit" + (and req + (str-contains? req "\"tool_choice\"") + (str-contains? req "\"name\":\"edit\"")) #t))) + (lambda () + (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) + (putenv "JCODE_FORCED_TOOL_NO_CALL_CHAR_LIMIT" (or old-limit "")) + (tcp-close srv)))) + +(let* ([sse-body "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"partial\"},\"finish_reason\":null}]}\n\n"] [srv (tcp-listen "127.0.0.1" 0)] [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))]