updates
ober
8352bef798c3b7910185364100742ab46ffaedc3
--- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -983,12 +983,32 @@ `(("Content-Type" . "application/json") ("Authorization" . ,(string-append "Bearer " key))))) +;; Stable per-process key for OpenAI prompt-cache routing: keeps a session's +;; requests on the same cache shard so server-side prefix caching hits more +;; reliably. Only sent to providers known to accept it — strict APIs +;; (e.g. mistral) 4xx on unknown fields. +(def *prompt-cache-key* (format "jcode-~a" (real-time))) + +(def (prompt-cache-key-provider? name) + (member name '("openai" "openrouter"))) + +(def (openai-cached-tokens usage) + ;; OpenAI-compat servers (openai, openrouter, vLLM, llama-server) report + ;; prefix-cache hits in prompt_tokens_details.cached_tokens. Absent on + ;; servers without prefix caching (e.g. ollama) — returns 0. + (let ((details (hash-get usage "prompt_tokens_details"))) + (if (and details (hash-table? details)) + (or (hash-get details "cached_tokens") 0) + 0))) + (def (openai-body provider messages tools) (let ((body (make-hash-table))) (hash-put! body "model" (provider-model provider)) (hash-put! body "max_tokens" 32768) (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (apply-logprobs! body) + (when (prompt-cache-key-provider? (provider-name provider)) + (hash-put! body "prompt_cache_key" *prompt-cache-key*)) (hash-put! body "messages" (map message->json messages)) (when (and tools (not (null? tools)) (not (model-rejects-tools? (provider-model provider)))) @@ -1146,7 +1166,11 @@ acc))))))) (def (anthropic-add-cache-control msg) - "Add cache_control to the last content block in a message." + "Add cache_control to the last content block in a message. Plain string + content (ordinary text turns) is promoted to a one-element text-block + list so it can carry a breakpoint too — otherwise only tool_use / + tool_result turns ever get cached and plain-text turns silently re-bill + the whole history." (let ((content (hash-get msg "content"))) (cond ((and (list? content) (not (null? content))) @@ -1154,6 +1178,13 @@ (when (hash-table? last-block) (hash-put! last-block "cache_control" (cache-control-ephemeral)))) msg) + ((and (string? content) (> (string-length content) 0)) + (let ((block (make-hash-table))) + (hash-put! block "type" "text") + (hash-put! block "text" content) + (hash-put! block "cache_control" (cache-control-ephemeral)) + (hash-put! msg "content" (list block))) + msg) (else msg)))) (def (find-system-message messages) @@ -1391,6 +1422,8 @@ (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (unless (skip-stream-logprobs? provider tools) (apply-logprobs! body)) + (when (prompt-cache-key-provider? (provider-name provider)) + (hash-put! body "prompt_cache_key" *prompt-cache-key*)) ;; Request usage data in stream (let ((opts (make-hash-table))) (hash-put! opts "include_usage" #t) @@ -1546,6 +1579,7 @@ (tool-calls . ,(length tool-calls)) (tokens-in . ,(or (hash-get usage-acc "prompt_tokens") 0)) (tokens-out . ,(or (hash-get usage-acc "completion_tokens") 0)) + (cache-read . ,(openai-cached-tokens usage-acc)) (cost . ,cost))) (when (tracing?) (log-trace logger "openai-stream-result" @@ -1557,6 +1591,7 @@ (values content tool-calls (list (cons 'tokens-in (or (hash-get usage-acc "prompt_tokens") 0)) (cons 'tokens-out (or (hash-get usage-acc "completion_tokens") 0)) + (cons 'cache-read (openai-cached-tokens usage-acc)) (cons 'cost cost)) (build-stats (unbox finish-reason-box) (unbox logprob-box) @@ -1565,10 +1600,11 @@ ;;; Anthropic Streaming ;;; (def (anthropic-stream-headers provider) + ;; Prompt caching is GA — the old prompt-caching beta header is obsolete + ;; (kept the non-stream path's header set in sync). `(("Content-Type" . "application/json") ("x-api-key" . ,(provider-api-key provider)) - ("anthropic-version" . "2023-06-01") - ("anthropic-beta" . "prompt-caching-2024-07-31"))) + ("anthropic-version" . "2023-06-01"))) (def (anthropic-stream-body provider messages tools) (let ((body (anthropic-body provider messages tools)))