Disable local template thinking for forced tools
ober
851b0b1e8c6897218f53a9e892e586f46ff9c0a1
--- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -1143,6 +1143,26 @@ (not (hash-get body "reasoning_effort"))) (hash-put! body "reasoning_effort" "low"))) +(def (ensure-chat-template-kwargs body) + (let ((existing (hash-get body "chat_template_kwargs"))) + (if (hash-table? existing) + existing + (let ((ctk (make-hash-table))) + (hash-put! body "chat_template_kwargs" ctk) + ctk)))) + +(def (apply-forced-tool-template-overrides! body provider) + ;; Some local OpenAI-compatible servers expose model thinking controls only + ;; through chat_template_kwargs. Exact forced-tool turns need the model to + ;; emit the structured call immediately; hidden reasoning drafts cannot be + ;; applied to files and quickly burn the benchmark wall-clock budget. + (when (and (local-provider? (provider-name provider)) + (current-tool-choice-override)) + (let ((ctk (ensure-chat-template-kwargs body))) + (hash-put! ctk "enable_thinking" #f) + (hash-put! ctk "reasoning_effort" "none") + (hash-put! ctk "thinking_budget" 0)))) + (def (mean-of lst) (and (pair? lst) (/ (apply + lst) (length lst) 1.0))) @@ -1625,6 +1645,7 @@ (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (apply-openai-provider-overrides! body provider) (apply-local-verified-overrides! body provider) + (apply-forced-tool-template-overrides! body provider) (maybe-apply-logprobs! body provider tools) (apply-prompt-cache-controls! body provider) (hash-put! body "messages" (map (lambda (m) (openai-message->json provider m)) messages)) @@ -2363,6 +2384,7 @@ (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (apply-openai-provider-overrides! body provider) (apply-local-verified-overrides! body provider) + (apply-forced-tool-template-overrides! body provider) (maybe-apply-logprobs! body provider tools) (apply-prompt-cache-controls! body provider) ;; Request usage data in stream --- a/test/run.ss +++ b/test/run.ss @@ -12787,7 +12787,13 @@ (provider-chat p (list (make-user-message "hi")) (list tool)))] [req (vector-ref captured 0)]) (check! "mlx forced tool defaults reasoning_effort low" - (and req (str-contains? req "\"reasoning_effort\":\"low\"")) #t))) + (and req (str-contains? req "\"reasoning_effort\":\"low\"")) #t) + (check! "mlx forced tool disables template thinking" + (and req + (str-contains? req "\"chat_template_kwargs\"") + (str-contains? req "\"enable_thinking\":false") + (str-contains? req "\"thinking_budget\":0") + (str-contains? req "\"reasoning_effort\":\"none\"")) #t))) (lambda () (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) (tcp-close srv)))) @@ -13110,7 +13116,13 @@ (lambda vals vals))] [req (vector-ref captured 0)]) (check! "mlx forced streaming tool defaults reasoning_effort low" - (and req (str-contains? req "\"reasoning_effort\":\"low\"")) #t))) + (and req (str-contains? req "\"reasoning_effort\":\"low\"")) #t) + (check! "mlx forced streaming tool disables template thinking" + (and req + (str-contains? req "\"chat_template_kwargs\"") + (str-contains? req "\"enable_thinking\":false") + (str-contains? req "\"thinking_budget\":0") + (str-contains? req "\"reasoning_effort\":\"none\"")) #t))) (lambda () (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) (tcp-close srv))))