Enable image viewing by default
ober
f00a361843532d89fcdcc85193fdd10c4816a5eb
--- a/docs/tools.md +++ b/docs/tools.md @@ -66,7 +66,12 @@ photos, and diagrams. It does not send pixels through the chat provider. Instead, it runs a configured vision backend and feeds that backend's text output back to the model as a tool result. -Configure the backend in `jcode.json` or `~/.jcode/config.json`: +`image_view` is enabled by default. If no `image_view` config is present, jcode +automatically uses the first available configured vision provider in this order: +OpenAI, Anthropic, Google/Gemini, then OpenRouter. If none of those are +configured, it falls back to the current provider when possible. + +You only need config to force a specific vision provider/model: ```json { @@ -80,11 +85,10 @@ Configure the backend in `jcode.json` or `~/.jcode/config.json`: } ``` -If `image_view.provider` is omitted, jcode uses the current configured provider. The built-in backend supports OpenAI-compatible providers, Anthropic, and -Google/Gemini. For unusual local vision CLIs, `image_view.argv` is still -available as an advanced override; it is an argument vector, not a shell string, -and supports `{path}`, `{question}`, and `{detail}` placeholders. +Google/Gemini. For unusual local vision CLIs, `image_view.argv` remains an +advanced override; it is an argument vector, not a shell string, and supports +`{path}`, `{question}`, and `{detail}` placeholders. ### Named agents (`task`'s `agent` parameter) --- a/src/jcode/tool/image.ss +++ b/src/jcode/tool/image.ss @@ -126,13 +126,46 @@ (def (image-provider-name) (or (config-ref "image_view" "provider") + (auto-image-provider-name) (config-provider))) (def (image-provider-model provider-name) (or (config-ref "image_view" "model") - (if (equal? provider-name (config-provider)) - (config-model) - (config-default-model provider-name)))) + (image-default-model provider-name) + (if (equal? provider-name (config-provider)) (config-model) #f) + (config-default-model provider-name))) + +(def (auto-image-provider-name) + (let ((current (config-provider))) + (or (first-ready-image-provider image-preferred-provider-order) + (and (image-provider-ready? current) current) + (and (local-provider? current) current)))) + +(def image-preferred-provider-order + ;; Prefer providers whose default model IDs here are known image-capable. + ;; Other OpenAI-compatible providers still work when explicitly configured + ;; via image_view.provider/image_view.model. + '("openai" "anthropic" "google" "openrouter")) + +(def (first-ready-image-provider names) + (let loop ((rest names)) + (cond + ((null? rest) #f) + ((image-provider-ready? (car rest)) (car rest)) + (else (loop (cdr rest)))))) + +(def (image-provider-ready? name) + (or (local-provider? name) + (let ((key (config-get-provider-key name))) + (and (string? key) (not (string=? key "")))))) + +(def (image-default-model provider-name) + (case (string->symbol (provider-kind provider-name)) + ((openai) "gpt-4o") + ((anthropic) "claude-sonnet-4-20250514") + ((google) "gemini-2.5-flash") + ((openrouter) "openai/gpt-4o") + (else #f))) (def (openai-image-observe provider path question detail) (let* ((url (string-append (provider-base-url provider) "/chat/completions")) --- a/test/run.ss +++ b/test/run.ss @@ -448,6 +448,7 @@ 'replace) (hashtable-set! image-cfg "argv" '("/bin/echo" "seen:{path}:{question}:{detail}")) (hashtable-set! image-cfg "timeout_seconds" 5) + (hashtable-set! image-cfg "allow_network" #t) (hashtable-set! cfg "image_view" image-cfg) (parameterize ((*config* cfg)) (check! "image argv placeholder rendering" @@ -472,15 +473,13 @@ [cfg (make-hashtable equal-hash equal?)] [providers (make-hashtable equal-hash equal?)] [openai (make-hashtable equal-hash equal?)] - [image-cfg (make-hashtable equal-hash equal?)] [body "{\"choices\":[{\"message\":{\"content\":\"builtin vision ok\"}}]}"]) (hashtable-set! openai "api_key" "unit-key") (hashtable-set! openai "base_url" base-url) (hashtable-set! providers "openai" openai) - (hashtable-set! image-cfg "provider" "openai") - (hashtable-set! image-cfg "model" "vision-unit") + (hashtable-set! cfg "provider" "mlx") + (hashtable-set! cfg "model" "text-local") (hashtable-set! cfg "providers" providers) - (hashtable-set! cfg "image_view" image-cfg) (dynamic-wind (lambda () (void)) (lambda () @@ -494,12 +493,13 @@ (lambda (s) (and (str-contains? s "builtin vision ok") (str-contains? s "Provider: openai") - (str-contains? s "Model: vision-unit")))) + (str-contains? s "Model: gpt-4o")))) (check-pred! "image_view built-in sends image_url data URL" req (lambda (s) (and (str-contains? s "\"type\":\"image_url\"") - (str-contains? s "data:image/png;base64,"))))))) + (str-contains? s "data:image/png;base64,") + (str-contains? s "\"model\":\"gpt-4o\""))))))) (lambda () (tcp-close srv))))) ;; ── Bash tool tests ───────────────────────────────────────────────