Add 6 new providers, Anthropic prompt caching, and tool_choice support
ober
0f82e99d2e73ea0bb33178e1a35f86d52e8f4885
--- a/lib/jcode/core/config.sls +++ b/lib/jcode/core/config.sls @@ -48,11 +48,14 @@ (make-hash-table))]) (hash-put! p "api_key" key) (hash-put! providers (cdr pair) p))))) - '(("OPENAI_API_KEY" . "openai") - ("ANTHROPIC_API_KEY" . "anthropic") + '(("OPENAI_API_KEY" . "openai") ("ANTHROPIC_API_KEY" . "anthropic") ("GOOGLE_API_KEY" . "google") ("OPENROUTER_API_KEY" . "openrouter") - ("DEEPSEEK_API_KEY" . "deepseek"))) + ("DEEPSEEK_API_KEY" . "deepseek") ("XAI_API_KEY" . "xai") + ("GROQ_API_KEY" . "groq") ("MISTRAL_API_KEY" . "mistral") + ("TOGETHER_API_KEY" . "together") + ("CEREBRAS_API_KEY" . "cerebras") + ("PERPLEXITY_API_KEY" . "perplexity"))) (hash-put! config "providers" providers) config)) (def (merge-opencode-keys! providers) @@ -79,11 +82,9 @@ "api_key" (hash-ref entry "key")) (hash-put! providers name p)))))) - '("openrouter" - "deepseek" - "google" - "openai" - "anthropic"))) + '("openrouter" "deepseek" "google" "openai" "anthropic" "xai" + "groq" "mistral" "together" "cerebras" + "perplexity"))) (catch (e) (void)))))) (def (config-ref . keys) (let loop ([obj (*config*)] [keys keys]) @@ -103,11 +104,9 @@ (config-detect-provider) "anthropic")) (def (config-detect-provider) - (let loop ([names '("anthropic" - "openrouter" - "openai" - "deepseek" - "google")]) + (let loop ([names '("anthropic" "openrouter" "openai" "deepseek" "google" "xai" + "groq" "mistral" "together" "cerebras" + "perplexity")]) (cond [(null? names) #f] [(config-get-provider-key (car names)) (car names)] --- a/lib/jcode/core/models.sls +++ b/lib/jcode/core/models.sls @@ -13,8 +13,8 @@ (std text json) (std os path) (jerboa core) (jerboa runtime)) (def (all-providers) - '("anthropic" "openai" "openrouter" "deepseek" "google" - "ollama")) + '("anthropic" "openai" "openrouter" "deepseek" "google" "ollama" "xai" + "groq" "mistral" "together" "cerebras" "perplexity")) (def (provider-display-name p) (case (string->symbol p) [(anthropic) "Anthropic"] @@ -23,6 +23,12 @@ [(deepseek) "DeepSeek"] [(google) "Google"] [(ollama) "Ollama (local)"] + [(xai) "xAI (Grok)"] + [(groq) "Groq"] + [(mistral) "Mistral"] + [(together) "Together AI"] + [(cerebras) "Cerebras"] + [(perplexity) "Perplexity"] [else p])) (def (provider-default-model provider) (case (string->symbol provider) @@ -32,6 +38,12 @@ [(deepseek) "deepseek-chat"] [(google) "gemini-2.5-flash"] [(ollama) "llama3.2"] + [(xai) "grok-3-mini"] + [(groq) "llama-3.3-70b-versatile"] + [(mistral) "mistral-large-latest"] + [(together) "meta-llama/Llama-3.3-70B-Instruct-Turbo"] + [(cerebras) "llama-3.3-70b"] + [(perplexity) "sonar-pro"] [else "gpt-4o"])) (def (provider-models provider) (unless (*models-cache*) (load-models-cache!)) @@ -48,6 +60,12 @@ [(google) google-models] [(openrouter) openrouter-models] [(ollama) ollama-models] + [(xai) xai-models] + [(groq) groq-models] + [(mistral) mistral-models] + [(together) together-models] + [(cerebras) cerebras-models] + [(perplexity) perplexity-models] [else '()])) (def *models-cache* (make-parameter #f)) (def (models-cache-path) @@ -142,4 +160,40 @@ '(("llama3.2" . "Llama 3.2") ("llama3.1" . "Llama 3.1") ("qwen2.5-coder" . "Qwen 2.5 Coder") ("deepseek-r1" . "DeepSeek R1") ("mistral" . "Mistral") - ("codellama" . "Code Llama") ("gemma2" . "Gemma 2")))) + ("codellama" . "Code Llama") ("gemma2" . "Gemma 2"))) + (def xai-models + '(("grok-3" . "Grok 3") + ("grok-3-mini" . "Grok 3 Mini") + ("grok-2" . "Grok 2"))) + (def groq-models + '(("llama-3.3-70b-versatile" . "Llama 3.3 70B") + ("llama-3.1-8b-instant" . "Llama 3.1 8B Instant") + ("gemma2-9b-it" . "Gemma 2 9B") + ("mixtral-8x7b-32768" . "Mixtral 8x7B") + ("deepseek-r1-distill-llama-70b" + . + "DeepSeek R1 Distill 70B"))) + (def mistral-models + '(("mistral-large-latest" . "Mistral Large") + ("mistral-medium-latest" . "Mistral Medium") + ("codestral-latest" . "Codestral") + ("devstral-small-latest" . "Devstral Small") + ("mistral-small-latest" . "Mistral Small"))) + (def together-models + '(("meta-llama/Llama-3.3-70B-Instruct-Turbo" + . + "Llama 3.3 70B Turbo") + ("deepseek-ai/DeepSeek-R1" . "DeepSeek R1") + ("Qwen/Qwen2.5-Coder-32B-Instruct" . "Qwen 2.5 Coder 32B") + ("meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo" + . + "Llama 3.1 405B Turbo"))) + (def cerebras-models + '(("llama-3.3-70b" . "Llama 3.3 70B") + ("llama-3.1-8b" . "Llama 3.1 8B") + ("qwen-2.5-32b" . "Qwen 2.5 32B"))) + (def perplexity-models + '(("sonar-pro" . "Sonar Pro") + ("sonar" . "Sonar") + ("sonar-reasoning-pro" . "Sonar Reasoning Pro") + ("sonar-reasoning" . "Sonar Reasoning")))) --- a/lib/jcode/provider/provider.sls +++ b/lib/jcode/provider/provider.sls @@ -51,6 +51,12 @@ [(openrouter) "https://openrouter.ai/api/v1"] [(deepseek) "https://api.deepseek.com/v1"] [(ollama) "http://localhost:11434/v1"] + [(xai) "https://api.x.ai/v1"] + [(groq) "https://api.groq.com/openai/v1"] + [(mistral) "https://api.mistral.ai/v1"] + [(together) "https://api.together.xyz/v1"] + [(cerebras) "https://api.cerebras.ai/v1"] + [(perplexity) "https://api.perplexity.ai"] [else (error 'provider-default-url "Unknown provider" name)])) (def (provider-chat provider messages tools) @@ -68,7 +74,8 @@ (api-call-with-retry (lambda () (case (string->symbol (provider-name provider)) - [(openai openrouter deepseek) + [(openai openrouter deepseek xai groq mistral together + cerebras perplexity) (openai-chat provider messages tools)] [(anthropic) (anthropic-chat provider messages tools)] [(google) (google-chat provider messages tools)] @@ -349,7 +356,8 @@ (hash-put! body "model" (provider-model provider)) (hash-put! body "messages" (map message->json messages)) (when (and tools (not (null? tools))) - (hash-put! body "tools" tools)) + (hash-put! body "tools" tools) + (hash-put! body "tool_choice" "auto")) body)) (def (openai-parse-response json) (let* ([choices (hash-ref json "choices" '())] @@ -376,7 +384,8 @@ (def (anthropic-headers provider) `(("Content-Type" . "application/json") ("x-api-key" . ,(provider-api-key provider)) - ("anthropic-version" . "2023-06-01"))) + ("anthropic-version" . "2023-06-01") + ("anthropic-beta" . "prompt-caching-2024-07-31"))) (def (anthropic-body provider messages tools) (let ([body (make-hash-table)] [system-msg (find-system-message messages)] @@ -384,14 +393,59 @@ (hash-put! body "model" (provider-model provider)) (hash-put! body "max_tokens" 8192) (when system-msg - (hash-put! body "system" (message-content system-msg))) - (hash-put! - body - "messages" - (map anthropic-convert-message other-msgs)) + (let ([block (make-hash-table)]) + (hash-put! block "type" "text") + (hash-put! block "text" (message-content system-msg)) + (hash-put! block "cache_control" (cache-control-ephemeral)) + (hash-put! body "system" (list block)))) + (let ([converted (map anthropic-convert-message + other-msgs)]) + (hash-put! + body + "messages" + (anthropic-apply-caching converted))) (when (and tools (not (null? tools))) - (hash-put! body "tools" (map anthropic-convert-tool tools))) + (hash-put! body "tools" (map anthropic-convert-tool tools)) + (hash-put! + body + "tool_choice" + (let ([tc (make-hash-table)]) + (hash-put! tc "type" "auto") + tc))) body)) + (def (cache-control-ephemeral) + (let ([cc (make-hash-table)]) + (hash-put! cc "type" "ephemeral") + cc)) + (def (anthropic-apply-caching messages) + "Add cache_control to the last 2 messages for Anthropic prompt caching." + (let* ([len (length messages)] + [cache-start (max 0 (- len 2))]) + (let loop ([msgs messages] [i 0] [acc '()]) + (if (null? msgs) + (reverse acc) + (let ([msg (car msgs)]) + (loop + (cdr msgs) + (+ i 1) + (cons + (if (>= i cache-start) + (anthropic-add-cache-control msg) + msg) + acc))))))) + (def (anthropic-add-cache-control msg) + "Add cache_control to the last content block in a message." + (let ([content (hash-get msg "content")]) + (cond + [(and (list? content) (not (null? content))) + (let* ([last-block (car (reverse content))]) + (when (hash-table? last-block) + (hash-put! + last-block + "cache_control" + (cache-control-ephemeral)))) + msg] + [else msg]))) (def (find-system-message messages) (find (lambda (m) (equal? (message-role m) "system")) @@ -604,7 +658,8 @@ (hash-put! body "stream_options" opts)) (hash-put! body "messages" (map message->json messages)) (when (and tools (not (null? tools))) - (hash-put! body "tools" tools)) + (hash-put! body "tools" tools) + (hash-put! body "tool_choice" "auto")) body)) (def (openai-stream-chat provider messages tools token-cb) (let* ([url (string-append @@ -780,7 +835,8 @@ (def (anthropic-stream-headers provider) `(("Content-Type" . "application/json") ("x-api-key" . ,(provider-api-key provider)) - ("anthropic-version" . "2023-06-01"))) + ("anthropic-version" . "2023-06-01") + ("anthropic-beta" . "prompt-caching-2024-07-31"))) (def (anthropic-stream-body provider messages tools) (let ([body (anthropic-body provider messages tools)]) (hash-put! body "stream" #t) @@ -989,7 +1045,8 @@ (model . ,(provider-model provider)) (messages . ,(length messages)))) (case (string->symbol (provider-name provider)) - [(openai openrouter deepseek ollama) + [(openai openrouter deepseek ollama xai groq mistral + together cerebras perplexity) (openai-stream-chat provider messages tools token-cb)] [(anthropic) (anthropic-stream-chat provider messages tools token-cb)] --- a/src/jcode/core/config.ss +++ b/src/jcode/core/config.ss @@ -55,11 +55,17 @@ (let ((p (or (hash-get providers (cdr pair)) (make-hash-table)))) (hash-put! p "api_key" key) (hash-put! providers (cdr pair) p))))) - '(("OPENAI_API_KEY" . "openai") - ("ANTHROPIC_API_KEY" . "anthropic") - ("GOOGLE_API_KEY" . "google") - ("OPENROUTER_API_KEY" . "openrouter") - ("DEEPSEEK_API_KEY" . "deepseek"))) + '(("OPENAI_API_KEY" . "openai") + ("ANTHROPIC_API_KEY" . "anthropic") + ("GOOGLE_API_KEY" . "google") + ("OPENROUTER_API_KEY" . "openrouter") + ("DEEPSEEK_API_KEY" . "deepseek") + ("XAI_API_KEY" . "xai") + ("GROQ_API_KEY" . "groq") + ("MISTRAL_API_KEY" . "mistral") + ("TOGETHER_API_KEY" . "together") + ("CEREBRAS_API_KEY" . "cerebras") + ("PERPLEXITY_API_KEY" . "perplexity"))) (hash-put! config "providers" providers) config)) @@ -80,7 +86,8 @@ (unless (hash-get p "api_key") ;; don't override existing (hash-put! p "api_key" (hash-ref entry "key")) (hash-put! providers name p)))))) - '("openrouter" "deepseek" "google" "openai" "anthropic"))) + '("openrouter" "deepseek" "google" "openai" "anthropic" + "xai" "groq" "mistral" "together" "cerebras" "perplexity"))) (catch (e) (void)))))) (def (config-ref . keys) @@ -107,7 +114,8 @@ (def (config-detect-provider) ;; Auto-detect first provider with a configured key - (let loop ((names '("anthropic" "openrouter" "openai" "deepseek" "google"))) + (let loop ((names '("anthropic" "openrouter" "openai" "deepseek" "google" + "xai" "groq" "mistral" "together" "cerebras" "perplexity"))) (cond ((null? names) #f) ((config-get-provider-key (car names)) (car names)) --- a/src/jcode/core/models.ss +++ b/src/jcode/core/models.ss @@ -16,7 +16,8 @@ ;; ---- Provider metadata ---- (def (all-providers) - '("anthropic" "openai" "openrouter" "deepseek" "google" "ollama")) + '("anthropic" "openai" "openrouter" "deepseek" "google" "ollama" + "xai" "groq" "mistral" "together" "cerebras" "perplexity")) (def (provider-display-name p) (case (string->symbol p) @@ -26,6 +27,12 @@ ((deepseek) "DeepSeek") ((google) "Google") ((ollama) "Ollama (local)") + ((xai) "xAI (Grok)") + ((groq) "Groq") + ((mistral) "Mistral") + ((together) "Together AI") + ((cerebras) "Cerebras") + ((perplexity) "Perplexity") (else p))) (def (provider-default-model provider) @@ -36,6 +43,12 @@ ((deepseek) "deepseek-chat") ((google) "gemini-2.5-flash") ((ollama) "llama3.2") + ((xai) "grok-3-mini") + ((groq) "llama-3.3-70b-versatile") + ((mistral) "mistral-large-latest") + ((together) "meta-llama/Llama-3.3-70B-Instruct-Turbo") + ((cerebras) "llama-3.3-70b") + ((perplexity) "sonar-pro") (else "gpt-4o"))) ;; ---- Model lists per provider ---- @@ -57,6 +70,12 @@ ((google) google-models) ((openrouter) openrouter-models) ((ollama) ollama-models) + ((xai) xai-models) + ((groq) groq-models) + ((mistral) mistral-models) + ((together) together-models) + ((cerebras) cerebras-models) + ((perplexity) perplexity-models) (else '()))) ;; ---- On-disk cache ---- @@ -183,3 +202,39 @@ ("mistral" . "Mistral") ("codellama" . "Code Llama") ("gemma2" . "Gemma 2"))) + +(def xai-models + '(("grok-3" . "Grok 3") + ("grok-3-mini" . "Grok 3 Mini") + ("grok-2" . "Grok 2"))) + +(def groq-models + '(("llama-3.3-70b-versatile" . "Llama 3.3 70B") + ("llama-3.1-8b-instant" . "Llama 3.1 8B Instant") + ("gemma2-9b-it" . "Gemma 2 9B") + ("mixtral-8x7b-32768" . "Mixtral 8x7B") + ("deepseek-r1-distill-llama-70b" . "DeepSeek R1 Distill 70B"))) + +(def mistral-models + '(("mistral-large-latest" . "Mistral Large") + ("mistral-medium-latest" . "Mistral Medium") + ("codestral-latest" . "Codestral") + ("devstral-small-latest" . "Devstral Small") + ("mistral-small-latest" . "Mistral Small"))) + +(def together-models + '(("meta-llama/Llama-3.3-70B-Instruct-Turbo" . "Llama 3.3 70B Turbo") + ("deepseek-ai/DeepSeek-R1" . "DeepSeek R1") + ("Qwen/Qwen2.5-Coder-32B-Instruct" . "Qwen 2.5 Coder 32B") + ("meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo" . "Llama 3.1 405B Turbo"))) + +(def cerebras-models + '(("llama-3.3-70b" . "Llama 3.3 70B") + ("llama-3.1-8b" . "Llama 3.1 8B") + ("qwen-2.5-32b" . "Qwen 2.5 32B"))) + +(def perplexity-models + '(("sonar-pro" . "Sonar Pro") + ("sonar" . "Sonar") + ("sonar-reasoning-pro" . "Sonar Reasoning Pro") + ("sonar-reasoning" . "Sonar Reasoning"))) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -58,6 +58,12 @@ ((openrouter) "https://openrouter.ai/api/v1") ((deepseek) "https://api.deepseek.com/v1") ((ollama) "http://localhost:11434/v1") + ((xai) "https://api.x.ai/v1") + ((groq) "https://api.groq.com/openai/v1") + ((mistral) "https://api.mistral.ai/v1") + ((together) "https://api.together.xyz/v1") + ((cerebras) "https://api.cerebras.ai/v1") + ((perplexity) "https://api.perplexity.ai") (else (error 'provider-default-url "Unknown provider" name)))) (def (provider-chat provider messages tools) @@ -72,10 +78,11 @@ (api-call-with-retry (lambda () (case (string->symbol (provider-name provider)) - ((openai openrouter deepseek) (openai-chat provider messages tools)) - ((anthropic) (anthropic-chat provider messages tools)) - ((google) (google-chat provider messages tools)) - ((ollama) (ollama-chat provider messages tools)) + ((openai openrouter deepseek xai groq mistral together cerebras perplexity) + (openai-chat provider messages tools)) + ((anthropic) (anthropic-chat provider messages tools)) + ((google) (google-chat provider messages tools)) + ((ollama) (ollama-chat provider messages tools)) (else (error 'provider-chat "Unknown provider" (provider-name provider))))))) (def (provider-stream provider messages tools callback) @@ -339,7 +346,8 @@ (hash-put! body "model" (provider-model provider)) (hash-put! body "messages" (map message->json messages)) (when (and tools (not (null? tools))) - (hash-put! body "tools" tools)) + (hash-put! body "tools" tools) + (hash-put! body "tool_choice" "auto")) body)) (def (openai-parse-response json) @@ -364,7 +372,8 @@ (def (anthropic-headers provider) `(("Content-Type" . "application/json") ("x-api-key" . ,(provider-api-key provider)) - ("anthropic-version" . "2023-06-01"))) + ("anthropic-version" . "2023-06-01") + ("anthropic-beta" . "prompt-caching-2024-07-31"))) (def (anthropic-body provider messages tools) (let ((body (make-hash-table)) @@ -372,13 +381,53 @@ (other-msgs (remove-system-messages messages))) (hash-put! body "model" (provider-model provider)) (hash-put! body "max_tokens" 8192) + ;; System message with prompt caching (when system-msg - (hash-put! body "system" (message-content system-msg))) - (hash-put! body "messages" (map anthropic-convert-message other-msgs)) + (let ((block (make-hash-table))) + (hash-put! block "type" "text") + (hash-put! block "text" (message-content system-msg)) + (hash-put! block "cache_control" (cache-control-ephemeral)) + (hash-put! body "system" (list block)))) + ;; Convert messages and apply caching to last 2 turns + (let ((converted (map anthropic-convert-message other-msgs))) + (hash-put! body "messages" (anthropic-apply-caching converted))) (when (and tools (not (null? tools))) - (hash-put! body "tools" (map anthropic-convert-tool tools))) + (hash-put! body "tools" (map anthropic-convert-tool tools)) + (hash-put! body "tool_choice" (let ((tc (make-hash-table))) + (hash-put! tc "type" "auto") + tc))) body)) +(def (cache-control-ephemeral) + (let ((cc (make-hash-table))) + (hash-put! cc "type" "ephemeral") + cc)) + +(def (anthropic-apply-caching messages) + "Add cache_control to the last 2 messages for Anthropic prompt caching." + (let* ((len (length messages)) + (cache-start (max 0 (- len 2)))) + (let loop ((msgs messages) (i 0) (acc '())) + (if (null? msgs) + (reverse acc) + (let ((msg (car msgs))) + (loop (cdr msgs) (+ i 1) + (cons (if (>= i cache-start) + (anthropic-add-cache-control msg) + msg) + acc))))))) + +(def (anthropic-add-cache-control msg) + "Add cache_control to the last content block in a message." + (let ((content (hash-get msg "content"))) + (cond + ((and (list? content) (not (null? content))) + (let* ((last-block (car (reverse content)))) + (when (hash-table? last-block) + (hash-put! last-block "cache_control" (cache-control-ephemeral)))) + msg) + (else msg)))) + (def (find-system-message messages) (find (lambda (m) (equal? (message-role m) "system")) messages)) @@ -564,7 +613,8 @@ (hash-put! body "stream_options" opts)) (hash-put! body "messages" (map message->json messages)) (when (and tools (not (null? tools))) - (hash-put! body "tools" tools)) + (hash-put! body "tools" tools) + (hash-put! body "tool_choice" "auto")) body)) (def (openai-stream-chat provider messages tools token-cb) @@ -663,7 +713,8 @@ (def (anthropic-stream-headers provider) `(("Content-Type" . "application/json") ("x-api-key" . ,(provider-api-key provider)) - ("anthropic-version" . "2023-06-01"))) + ("anthropic-version" . "2023-06-01") + ("anthropic-beta" . "prompt-caching-2024-07-31"))) (def (anthropic-stream-body provider messages tools) (let ((body (anthropic-body provider messages tools))) @@ -778,8 +829,9 @@ (model . ,(provider-model provider)) (messages . ,(length messages)))) (case (string->symbol (provider-name provider)) - ((openai openrouter deepseek ollama) (openai-stream-chat provider messages tools token-cb)) - ((anthropic) (anthropic-stream-chat provider messages tools token-cb)) + ((openai openrouter deepseek ollama xai groq mistral together cerebras perplexity) + (openai-stream-chat provider messages tools token-cb)) + ((anthropic) (anthropic-stream-chat provider messages tools token-cb)) ((google) ;; Google: non-streaming fallback (let* ((response (provider-chat provider messages tools))