Lower reasoning on local forced tool calls
ober
1b52c89ea9b687c4c046ea5f518f05424acb1c09
--- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -1104,6 +1104,16 @@ (when v (hash-put! body field v)))) *openai-provider-request-fields*)) +(def (apply-local-forced-tool-overrides! body provider) + ;; Local reasoning models can spend a forced tool turn drafting code in + ;; reasoning_content before they emit the required function call. When jcode + ;; has already narrowed the schema and set tool_choice, ask for low reasoning + ;; effort unless the provider config made an explicit choice. + (when (and (local-provider? (provider-name provider)) + (current-tool-choice-override) + (not (hash-get body "reasoning_effort"))) + (hash-put! body "reasoning_effort" "low"))) + (def (mean-of lst) (and (pair? lst) (/ (apply + lst) (length lst) 1.0))) @@ -1571,6 +1581,7 @@ (hash-put! body "max_tokens" (openai-max-tokens provider)) (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (apply-openai-provider-overrides! body provider) + (apply-local-forced-tool-overrides! body provider) (maybe-apply-logprobs! body provider tools) (apply-prompt-cache-controls! body provider) (hash-put! body "messages" (map (lambda (m) (openai-message->json provider m)) messages)) @@ -2309,6 +2320,7 @@ (hash-put! body "max_tokens" (openai-max-tokens provider)) (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (apply-openai-provider-overrides! body provider) + (apply-local-forced-tool-overrides! body provider) (maybe-apply-logprobs! body provider tools) (apply-prompt-cache-controls! body provider) ;; Request usage data in stream --- a/test/run.ss +++ b/test/run.ss @@ -11352,6 +11352,35 @@ [srv (tcp-listen "127.0.0.1" 0)] [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] [captured (vector #f)] + [tool (make-hashtable equal-hash equal?)] + [fn (make-hashtable equal-hash equal?)] + [params (make-hashtable equal-hash equal?)] + [old-max-tokens (getenv "JCODE_MAX_TOKENS")]) + (hashtable-set! params "type" "object") + (hashtable-set! fn "name" "edit") + (hashtable-set! fn "description" "edit") + (hashtable-set! fn "parameters" params) + (hashtable-set! tool "type" "function") + (hashtable-set! tool "function" fn) + (dynamic-wind + (lambda () (putenv "JCODE_MAX_TOKENS" "")) + (lambda () + (serve-one-captured-json! srv captured 200 chat-body) + (let* ([p (make-provider "mlx" "" "deepseek-v4-pro" base-url)] + [_ (parameterize ([current-tool-choice-override + (openai-function-tool-choice "edit")]) + (provider-chat p (list (make-user-message "hi")) (list tool)))] + [req (vector-ref captured 0)]) + (check! "mlx forced tool defaults reasoning_effort low" + (and req (str-contains? req "\"reasoning_effort\":\"low\"")) #t))) + (lambda () + (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) + (tcp-close srv)))) + +(let* ([chat-body "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n"] + [srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] [cfg (make-hashtable equal-hash equal?)] [providers (make-hashtable equal-hash equal?)] [mlx (make-hashtable equal-hash equal?)] @@ -11630,6 +11659,46 @@ (tcp-close srv)))) (let* ([sse-body + (string-append + "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":null}]}\n\n" + "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + "data: [DONE]\n\n")] + [srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [tool (make-hashtable equal-hash equal?)] + [fn (make-hashtable equal-hash equal?)] + [params (make-hashtable equal-hash equal?)] + [old-max-tokens (getenv "JCODE_MAX_TOKENS")]) + (hashtable-set! params "type" "object") + (hashtable-set! fn "name" "edit") + (hashtable-set! fn "description" "edit") + (hashtable-set! fn "parameters" params) + (hashtable-set! tool "type" "function") + (hashtable-set! tool "function" fn) + (dynamic-wind + (lambda () (putenv "JCODE_MAX_TOKENS" "")) + (lambda () + (serve-one-captured-sse! srv captured sse-body) + (let* ([p (make-provider "mlx" "" "deepseek-v4-pro" base-url)] + [_ (call-with-values + (lambda () + (parameterize ([current-tool-choice-override + (openai-function-tool-choice "edit")]) + (provider-stream-chat + p + (list (make-user-message "hi")) + (list tool) + (lambda (_token) #f)))) + (lambda vals vals))] + [req (vector-ref captured 0)]) + (check! "mlx forced streaming tool defaults reasoning_effort low" + (and req (str-contains? req "\"reasoning_effort\":\"low\"")) #t))) + (lambda () + (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) + (tcp-close srv)))) + +(let* ([sse-body "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"partial\"},\"finish_reason\":null}]}\n\n"] [srv (tcp-listen "127.0.0.1" 0)] [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))]