Fix Ollama native tool history encoding
ober
c8373841d0ab372658270f64c16c2dae93d943a3
--- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -1433,13 +1433,23 @@ ;; jcode can recover the textual call. Retry those specific failures without ;; the structured tools field; the system prompt still lists tools and ;; jcode's text-tool-call recovery can lift text calls back into tool calls. + ;; + ;; Native /api/chat can fail earlier while rendering the tool schema through + ;; the model's template. Ollama reports that as HTTP 400 with a parser error + ;; such as "Value looks like object, but can't find closing '}' symbol". + ;; Treat that as the same compatibility class, but only callers with tools + ;; present use this predicate to retry without the structured tools field. (let ((msg (err->string e))) - (and (string-contains msg "API error 500") - (or (string-contains msg "unexpected EOF") - (string-contains msg "tool_call") - (and (string-contains msg "closed by") - (or (string-contains msg "function") - (string-contains msg "parameter"))))))) + (or + (and (string-contains msg "API error 500") + (or (string-contains msg "unexpected EOF") + (string-contains msg "tool_call") + (and (string-contains msg "closed by") + (or (string-contains msg "function") + (string-contains msg "parameter"))))) + (and (string-contains msg "API error 400") + (or (string-contains msg "Value looks like object") + (string-contains msg "can't find closing")))))) (def (short-error-string e) (let ((msg (err->string e))) @@ -1499,10 +1509,36 @@ (hash-for-each (lambda (k v) (hash-put! opts k v)) configured))) (if (pair? (hash-keys opts)) opts #f))) +(def (ollama-native-arguments-object args) + (cond + ((hash-table? args) args) + ((string? args) + (let ((parsed (guard (e [(error? e) #f]) + (string->json-object args)))) + (if (hash-table? parsed) parsed (make-hash-table)))) + (else (make-hash-table)))) + +(def (ollama-native-history-tool-call tc) + ;; OpenAI history stores function.arguments as a JSON string. Native Ollama's + ;; chat template expects an object here; Qwen templates reject the string form. + (when (hash-table? tc) + (let ((fn (hash-get tc "function"))) + (when (hash-table? fn) + (hash-put! fn "arguments" + (ollama-native-arguments-object (hash-get fn "arguments")))))) + tc) + +(def (ollama-native-message->json msg) + (let ((j (message->json msg))) + (let ((tcs (hash-get j "tool_calls"))) + (when (and tcs (list? tcs)) + (hash-put! j "tool_calls" (map ollama-native-history-tool-call tcs)))) + j)) + (def (ollama-native-body provider messages tools stream?) (let ((body (make-hash-table))) (hash-put! body "model" (provider-model provider)) - (hash-put! body "messages" (map message->json messages)) + (hash-put! body "messages" (map ollama-native-message->json messages)) (hash-put! body "stream" stream?) (hash-put! body "keep_alive" (ollama-keep-alive)) (let ((opts (ollama-sampling-options provider))) @@ -1912,19 +1948,18 @@ (when (and event-type data-str) (let ((json (guard (e [(error? e) #f]) (string->json-object data-str)))) (when (and json (hash-table? json)) - (cond - ;; Text delta - ((equal? event-type "content_block_delta") + (case (if (string? event-type) (string->symbol event-type) 'unknown) + ((content_block_delta) (let ((delta (hash-get json "delta"))) - (when delta + (when (and delta (hash-table? delta)) (let ((dtype (hash-get delta "type"))) - (cond - ((equal? dtype "text_delta") + (case (if (string? dtype) (string->symbol dtype) 'unknown) + ((text_delta) (let ((text (hash-get delta "text"))) (when text (put-string text-acc text) (token-cb text)))) - ((equal? dtype "input_json_delta") + ((input_json_delta) ;; Accumulate tool input (let ((idx (current-idx)) (partial (hash-get delta "partial_json"))) @@ -1934,9 +1969,10 @@ (hash-put! acc "args" (string-append (or (hash-get acc "args") "") - partial))))))))))) + partial))))))) + (else (void))))))) ;; Tool use block start - ((equal? event-type "content_block_start") + ((content_block_start) (let ((block (hash-get json "content_block"))) (when (and block (equal? (hash-get block "type") "tool_use")) (let ((idx (hash-get json "index"))) @@ -1946,16 +1982,16 @@ (hash-put! acc "name" (hash-get block "name")) (hash-put! tu-table idx acc)))))) ;; Block ended - ((equal? event-type "content_block_stop") + ((content_block_stop) (current-idx #f)) ;; Usage from message lifecycle events - ((equal? event-type "message_start") + ((message_start) (let ((msg (hash-get json "message"))) (when msg (let ((usage (hash-get msg "usage"))) (when (and usage (hash-table? usage)) (hash-for-each (lambda (k v) (hash-put! usage-acc k v)) usage)))))) - ((equal? event-type "message_delta") + ((message_delta) ;; delta.stop_reason: "end_turn" | "max_tokens" | ;; "stop_sequence" | "tool_use". We map max_tokens to ;; the OpenAI-style "length" so detect-truncated fires. @@ -1968,7 +2004,7 @@ (let ((usage (hash-get json "usage"))) (when (and usage (hash-table? usage)) (hash-for-each (lambda (k v) (hash-put! usage-acc k v)) usage)))) - (#t (void))))))))))))) + (else (void)))))))))))) (unless (= http-status 200) (log-error logger "stream-http-error" `((status . ,http-status) (url . ,url))))) --- a/test/run.ss +++ b/test/run.ss @@ -211,6 +211,58 @@ (close-port out) (close-port in))))))) +(define (serve-one-dynamic-json! srv captured-body responder) + (fork-thread + (lambda () + (let-values ([(in out) (tcp-accept srv)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (let* ([req (read-test-http-request in)] + [resp (responder req)] + [status (car resp)] + [body (cdr resp)]) + (vector-set! captured-body 0 req) + (put-string out + (string-append + "HTTP/1.1 " (number->string status) " OK\r\n" + "Content-Type: application/json\r\n" + "Content-Length: " (number->string (string-length body)) "\r\n" + "Connection: close\r\n" + "\r\n" + body)) + (flush-output-port out))) + (lambda () + (close-port out) + (close-port in))))))) + +(define (serve-two-captured-json! srv captured-a status-a body-a captured-b status-b body-b) + (fork-thread + (lambda () + (let accept-one ([captured captured-a] + [status status-a] + [body body-a] + [rest (list captured-b status-b body-b)]) + (let-values ([(in out) (tcp-accept srv)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (vector-set! captured 0 (read-test-http-request in)) + (put-string out + (string-append + "HTTP/1.1 " (number->string status) " OK\r\n" + "Content-Type: application/json\r\n" + "Content-Length: " (number->string (string-length body)) "\r\n" + "Connection: close\r\n" + "\r\n" + body)) + (flush-output-port out)) + (lambda () + (close-port out) + (close-port in)))) + (unless (null? rest) + (accept-one (car rest) (cadr rest) (caddr rest) '())))))) + ;; ── Setup ───────────────────────────────────────────────────────── (current-log-level 'warn) @@ -5047,6 +5099,164 @@ (lambda (x) (and (number? x) (> x 0)))))) (lambda () (tcp-close srv)))) +;; ── provider: interface smoke tests ─────────────────────────────── + +(section "=== provider: interface smoke tests ===") + +(define openai-hi-sse + (string-append + "data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"unit\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"hi\"},\"finish_reason\":null}]}\n\n" + "data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"unit\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + "data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"unit\",\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":1}}\n\n" + "data: [DONE]\n\n")) + +(define ollama-hi-ndjson + (string-append + "{\"model\":\"unit\",\"message\":{\"role\":\"assistant\",\"content\":\"hi\"},\"done\":false}\n" + "{\"model\":\"unit\",\"done\":true,\"prompt_eval_count\":3,\"eval_count\":1}\n")) + +(define anthropic-hi-sse + (string-append + "event: message_start\n" + "data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"unit\",\"stop_reason\":null,\"usage\":{\"input_tokens\":3,\"output_tokens\":0}}}\n\n" + "event: content_block_delta\n" + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hi\"}}\n\n" + "event: message_delta\n" + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\n" + "event: message_stop\n" + "data: {\"type\":\"message_stop\"}\n\n")) + +(define google-hi-json + "{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"hi\"}]}}],\"usageMetadata\":{\"promptTokenCount\":3,\"candidatesTokenCount\":1,\"totalTokenCount\":4}}") + +(define grok-responses-hi-sse + (string-append + "event: response.output_text.delta\n" + "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n" + "event: response.completed\n" + "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":3,\"output_tokens\":1},\"output\":[]}}\n\n")) + +(define (stream-chat-result provider messages tools) + (call-with-values + (lambda () + (provider-stream-chat provider messages tools (lambda (token) #f))) + (lambda (content tcs usage) (list content tcs usage)))) + +(define (check-usage desc usage in out) + (check! (string-append desc " tokens-in") (cdr (assoc 'tokens-in usage)) in) + (check! (string-append desc " tokens-out") (cdr (assoc 'tokens-out usage)) out)) + +(define (run-openai-compatible-hi-smoke provider-name) + (let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-sse! srv captured openai-hi-sse) + (let* ([p (make-provider provider-name "unit-key" "unit-model" base-url)] + [result (stream-chat-result p (list (make-user-message "hi")) '())] + [req (vector-ref captured 0)] + [desc (string-append provider-name " openai-compatible hi")]) + (check! desc (car result) "hi") + (check! (string-append desc " sends messages") + (and req (str-contains? req "\"messages\"")) #t) + (check-usage desc (list-ref result 2) 3 1))) + (lambda () (tcp-close srv))))) + +(for-each run-openai-compatible-hi-smoke + '("openai" "openrouter" "deepseek" "mlx" "xai" "groq" + "mistral" "together" "cerebras" "perplexity")) + +(let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-json! srv captured 200 ollama-hi-ndjson) + (let* ([p (make-provider "ollama" "" "unit-model" base-url)] + [result (stream-chat-result p (list (make-user-message "hi")) '())] + [req (vector-ref captured 0)]) + (check! "ollama native hi" (car result) "hi") + (check! "ollama native hi uses keep_alive" + (and req (str-contains? req "\"keep_alive\"")) #t) + (check-usage "ollama native hi" (list-ref result 2) 3 1))) + (lambda () (tcp-close srv)))) + +(let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [cfg (make-hashtable equal-hash equal?)] + [providers (make-hashtable equal-hash equal?)] + [ollama (make-hashtable equal-hash equal?)]) + (hashtable-set! ollama "wire" "openai") + (hashtable-set! providers "ollama" ollama) + (hashtable-set! cfg "providers" providers) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-sse! srv captured openai-hi-sse) + (let* ([p (make-provider "ollama" "" "unit-model" base-url)] + [result (parameterize ([*config* cfg]) + (stream-chat-result p (list (make-user-message "hi")) '()))] + [req (vector-ref captured 0)]) + (check! "ollama openai-compatible hi" (car result) "hi") + (check! "ollama openai-compatible hi omits keep_alive" + (and req (str-contains? req "\"keep_alive\"")) #f) + (check-usage "ollama openai-compatible hi" (list-ref result 2) 3 1))) + (lambda () (tcp-close srv)))) + +(let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-sse! srv captured anthropic-hi-sse) + (let* ([p (make-provider "anthropic" "unit-key" "unit-model" base-url)] + [result (stream-chat-result p (list (make-user-message "hi")) '())] + [req (vector-ref captured 0)]) + (check! "anthropic stream hi" (car result) "hi") + (check! "anthropic stream hi sends stream flag" + (and req (str-contains? req "\"stream\":true")) #t) + (check-usage "anthropic stream hi" (list-ref result 2) 3 1))) + (lambda () (tcp-close srv)))) + +(let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1beta" (tcp-server-port srv))] + [captured (vector #f)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-json! srv captured 200 google-hi-json) + (let* ([p (make-provider "google" "unit-key" "gemini-unit" base-url)] + [result (stream-chat-result p (list (make-user-message "hi")) '())] + [req (vector-ref captured 0)]) + (check! "google json hi" (car result) "hi") + (check! "google json hi sends contents" + (and req (str-contains? req "\"contents\"")) #t) + (check-usage "google json hi" (list-ref result 2) 3 1))) + (lambda () (tcp-close srv)))) + +(with-temp-home + (lambda () + (let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-sse! srv captured grok-responses-hi-sse) + (let* ([p (make-provider "grok" "unit-key" "grok-build" base-url)] + [result (stream-chat-result p (list (make-user-message "hi")) '())] + [req (vector-ref captured 0)]) + (check! "grok responses hi" (car result) "hi") + (check! "grok responses hi sends input" + (and req (str-contains? req "\"input\"")) #t) + (check-usage "grok responses hi" (list-ref result 2) 3 1))) + (lambda () (tcp-close srv)))))) + ;; ── provider: streaming HTTP error bodies ───────────────────────── (section "=== provider: streaming HTTP error bodies ===") @@ -5112,6 +5322,91 @@ (cdr (assoc 'cache-read (list-ref result 2))) 0))) (lambda () (tcp-close srv)))) +(let* ([ndjson-body (string-append + "{\"model\":\"unit\",\"message\":{\"role\":\"assistant\",\"content\":\"ok\"},\"done\":false}\n" + "{\"model\":\"unit\",\"done\":true,\"prompt_eval_count\":100,\"eval_count\":2}\n")] + [error-body "{\"error\":\"Value looks like object, but can't find closing '}' symbol\"}"] + [srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [tc (restore-tool-call "call_1" "hello" "{\"name\":\"user\"}")]) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-dynamic-json! srv captured + (lambda (req) + (if (str-contains? req "\"arguments\":\"{") + (cons 400 error-body) + (cons 200 ndjson-body)))) + (let* ([p (make-provider "ollama" "" "unit-test-model" base-url)] + [result (call-with-values + (lambda () + (provider-stream-chat + p + (list (make-user-message "hi") + (make-assistant-message #f (list tc)) + (make-tool-result "call_1" "Hello, user!") + (make-user-message "hi")) + '() + (lambda (token) #f))) + (lambda (content tcs usage) (list content tcs usage)))] + [req-json (string->json-object (vector-ref captured 0))] + [msgs (hashtable-ref req-json "messages" '())] + [asst (list-ref msgs 1)] + [tcs (hashtable-ref asst "tool_calls" '())] + [fn (hashtable-ref (car tcs) "function" #f)] + [args (and fn (hashtable-ref fn "arguments" #f))]) + (check! "ollama native history reply" (car result) "ok") + (check-pred! "ollama native history arguments object" args hashtable?) + (check! "ollama native history arguments name" + (and (hashtable? args) (hashtable-ref args "name" #f)) + "user"))) + (lambda () (tcp-close srv)))) + +(let* ([error-body "{\"error\":\"Value looks like object, but can't find closing '}' symbol\"}"] + [ndjson-body (string-append + "{\"model\":\"unit\",\"message\":{\"role\":\"assistant\",\"content\":\"ok\"},\"done\":false}\n" + "{\"model\":\"unit\",\"done\":true,\"prompt_eval_count\":100,\"eval_count\":2}\n")] + [srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured-a (vector #f)] + [captured-b (vector #f)] + [tool (make-hashtable equal-hash equal?)] + [fn (make-hashtable equal-hash equal?)] + [params (make-hashtable equal-hash equal?)]) + (hashtable-set! params "type" "object") + (hashtable-set! fn "name" "lookup") + (hashtable-set! fn "description" "lookup") + (hashtable-set! fn "parameters" params) + (hashtable-set! tool "type" "function") + (hashtable-set! tool "function" fn) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-two-captured-json! srv captured-a 400 error-body captured-b 200 ndjson-body) + (let* ([p (make-provider "ollama" "" "unit-test-model" base-url)] + [result (call-with-values + (lambda () + (provider-stream-chat + p + (list (make-user-message "hi")) + (list tool) + (lambda (token) #f))) + (lambda (content tcs usage) (list content tcs usage)))] + [req-a (vector-ref captured-a 0)] + [req-b (vector-ref captured-b 0)]) + (check! "ollama native stream retries tool-template 400 reply" + (car result) "ok") + (check! "ollama native stream retry first includes tools" + (and req-a (str-contains? req-a "\"tools\"")) #t) + (check! "ollama native stream retry second omits tools" + (and req-b (str-contains? req-b "\"tools\"")) #f) + (check! "ollama native stream retry tokens-in" + (cdr (assoc 'tokens-in (list-ref result 2))) 100) + (check! "ollama native stream retry tokens-out" + (cdr (assoc 'tokens-out (list-ref result 2))) 2))) + (lambda () (tcp-close srv)))) + (let* ([chat-body "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n"] [srv (tcp-listen "127.0.0.1" 0)] [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))]