updates
ober
7dea810a140d2b5a3e96a9fb5905f099afbb68e1
--- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -789,6 +789,15 @@ (hash-put! body "logprobs" #t) (hash-put! body "top_logprobs" 5))) +(def (skip-stream-logprobs? provider tools) + ;; Ollama's OpenAI-compatible endpoint rejects logprobs when a streaming + ;; request also carries structured tools. Keep logprobs for plain streaming + ;; and for providers that support the combination. + (and (equal? (provider-name provider) "ollama") + tools + (not (null? tools)) + (not (model-rejects-tools? (provider-model provider))))) + (def (mean-of lst) (and (pair? lst) (/ (apply + lst) (length lst) 1.0))) @@ -1380,7 +1389,8 @@ (hash-put! body "stream" #t) (hash-put! body "max_tokens" 32768) (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) - (apply-logprobs! body) + (unless (skip-stream-logprobs? provider tools) + (apply-logprobs! body)) ;; Request usage data in stream (let ((opts (make-hash-table))) (hash-put! opts "include_usage" #t) --- a/test/run.ss +++ b/test/run.ss @@ -112,11 +112,14 @@ (cond [(eof-object? line) (void)] [(or (string=? line "") (string=? line "\r")) - (let drain ([remaining content-length]) - (when (> remaining 0) - (let ([c (read-char in)]) - (unless (eof-object? c) - (drain (- remaining 1))))))] + (let ([body (open-output-string)]) + (let drain ([remaining content-length]) + (when (> remaining 0) + (let ([c (read-char in)]) + (unless (eof-object? c) + (write-char c body) + (drain (- remaining 1)))))) + (get-output-string body))] [else (let ([clean (string-trim-right line)]) (header-loop @@ -152,6 +155,27 @@ (close-port out) (close-port in))))))) +(define (serve-one-captured-sse! srv captured-body body) + (fork-thread + (lambda () + (let-values ([(in out) (tcp-accept srv)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (vector-set! captured-body 0 (read-test-http-request in)) + (put-string out + (string-append + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/event-stream\r\n" + "Content-Length: " (number->string (string-length body)) "\r\n" + "Connection: close\r\n" + "\r\n" + body)) + (flush-output-port out)) + (lambda () + (close-port out) + (close-port in))))))) + ;; ── Setup ───────────────────────────────────────────────────────── (current-log-level 'warn) @@ -1946,6 +1970,52 @@ (section "=== provider: streaming HTTP error bodies ===") +(let* ([sse-body (string-append + "data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"unit\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\n" + "data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"unit\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + "data: [DONE]\n\n")] + [srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [cfg (make-hashtable equal-hash equal?)] + [expert (make-hashtable equal-hash equal?)] + [esc (make-hashtable equal-hash equal?)] + [tool (make-hashtable equal-hash equal?)] + [fn (make-hashtable equal-hash equal?)] + [params (make-hashtable equal-hash equal?)]) + (hashtable-set! esc "request_logprobs" #t) + (hashtable-set! expert "escalation" esc) + (hashtable-set! cfg "expert" expert) + (hashtable-set! params "type" "object") + (hashtable-set! fn "name" "lookup") + (hashtable-set! fn "description" "lookup") + (hashtable-set! fn "parameters" params) + (hashtable-set! tool "type" "function") + (hashtable-set! tool "function" fn) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-sse! srv captured sse-body) + (let* ([p (make-provider "ollama" "" "unit-test-model" base-url)] + [reply (call-with-values + (lambda () + (parameterize ([*config* cfg]) + (provider-stream-chat + p + (list (make-user-message "hi")) + (list tool) + (lambda (token) #f)))) + (lambda (content tcs usage) content))] + [req (vector-ref captured 0)]) + (check! "ollama stream tools reply" reply "ok") + (check! "ollama stream tools include tools" + (and req (str-contains? req "\"tools\"")) #t) + (check! "ollama stream tools omit logprobs" + (and req (str-contains? req "\"logprobs\"")) #f) + (check! "ollama stream tools omit top_logprobs" + (and req (str-contains? req "\"top_logprobs\"")) #f))) + (lambda () (tcp-close srv)))) + (let* ([body "{\"error\":{\"message\":\"unexpected EOF while reading stream; context window exhausted while decoding response!!\"}}"] [srv (tcp-listen "127.0.0.1" 0)] [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))]