Use configured providers for image viewing
ober
11b0290a2ab95f27aeb851aa5a9bba44d3e56b96
--- a/docs/tools.md +++ b/docs/tools.md @@ -71,20 +71,20 @@ Configure the backend in `jcode.json` or `~/.jcode/config.json`: ```json { "image_view": { - "argv": ["/path/to/vision-adapter", "{path}", "{question}", "{detail}"], + "provider": "openai", + "model": "gpt-4o", "timeout_seconds": 120, "max_bytes": 20971520, - "allow_network": false, - "read_paths": ["/path/to/local/model/files"], - "env": [] + "max_tokens": 4096 } } ``` -`argv` is an argument vector, not a shell string. Supported placeholders are -`{path}`, `{question}`, and `{detail}`. If no placeholder is present, jcode -appends the image path and question as final arguments. The backend should print -a concise visual description, OCR text, or direct answer to stdout. +If `image_view.provider` is omitted, jcode uses the current configured provider. +The built-in backend supports OpenAI-compatible providers, Anthropic, and +Google/Gemini. For unusual local vision CLIs, `image_view.argv` is still +available as an advanced override; it is an argument vector, not a shell string, +and supports `{path}`, `{question}`, and `{detail}` placeholders. ### Named agents (`task`'s `agent` parameter) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -21,6 +21,7 @@ message->responses-input responses-body responses-headers + http-post-json google-usage->alist grok-backend grok-backend-from --- a/src/jcode/tool/image.ss +++ b/src/jcode/tool/image.ss @@ -8,9 +8,13 @@ (import :jerboa/core :jerboa/runtime + :std/text/json + (rename (only (std text base64) base64-encode) + (base64-encode std-base64-encode)) :std/misc/string :std/os/path (only (std os path-util) file-size) + (only (chezscheme) open-file-input-port get-bytevector-all) :std/os/platform (only (std os limits sandbox) sandbox-policy @@ -23,7 +27,9 @@ process-result-stdout process-result-stderr) :jcode/core/config + :jcode/core/models :jcode/core/log + :jcode/provider/provider :jcode/tool/registry) (def logger (make-logger "tool.image")) @@ -68,16 +74,15 @@ (image-max-bytes) path)) (else (let ((argv (image-view-argv path question detail))) - (if (not argv) - (image-view-config-error) - (run-image-view-backend path question detail argv))))))) + (if argv + (run-image-view-backend path question detail argv) + (run-image-provider-backend path question detail))))))) (def (image-view-config-error) (string-append - "Error: image_view is not configured.\n" - "Add an image_view.argv array to jcode.json or ~/.jcode/config.json, for example:\n" - " \"image_view\": { \"argv\": [\"/path/to/vision-adapter\", \"{path}\", \"{question}\"], \"timeout_seconds\": 120 }\n" - "The backend must print a textual observation to stdout.")) + "Error: image_view could not find a built-in vision provider.\n" + "Set image_view.provider/image_view.model in jcode.json or ~/.jcode/config.json, " + "or set image_view.argv as an advanced override.")) (def (run-image-view-backend path question detail argv) (log-info logger "execute" `((path . ,path) (argv0 . ,(car argv)))) @@ -96,6 +101,185 @@ question text)))))) +(def (run-image-provider-backend path question detail) + (let* ((provider-name (image-provider-name)) + (model-name (image-provider-model provider-name)) + (kind (provider-kind provider-name)) + (api-key (or (config-get-provider-key provider-name) ""))) + (cond + ((and (not (local-provider? provider-name)) + (string=? api-key "")) + (format "Error: image_view provider ~a has no API key configured. Set image_view.provider to a configured vision provider, or set image_view.argv." + provider-name)) + ((not model-name) + (format "Error: image_view provider ~a has no model configured" provider-name)) + (else + (let ((provider (make-provider provider-name api-key model-name))) + (case (string->symbol kind) + ((anthropic) (anthropic-image-observe provider path question detail)) + ((google) (google-image-observe provider path question detail)) + ((openai openrouter deepseek xai groq mistral together cerebras perplexity ollama mlx) + (openai-image-observe provider path question detail)) + (else + (format "Error: image_view provider kind ~a is not supported for built-in vision. Set image_view.argv for a custom backend." + kind)))))))) + +(def (image-provider-name) + (or (config-ref "image_view" "provider") + (config-provider))) + +(def (image-provider-model provider-name) + (or (config-ref "image_view" "model") + (if (equal? provider-name (config-provider)) + (config-model) + (config-default-model provider-name)))) + +(def (openai-image-observe provider path question detail) + (let* ((url (string-append (provider-base-url provider) "/chat/completions")) + (body (openai-image-body provider path question detail)) + (headers `(("Content-Type" . "application/json") + ("Authorization" . ,(string-append "Bearer " (image-provider-key provider))))) + (body-json (json-object->string body))) + (let-values (((status text) (http-post-json url headers body-json))) + (if (= status 200) + (let ((answer (extract-openai-image-text (string->json-object text)))) + (format-image-observation provider path question detail answer)) + (format "Error: image_view provider ~a returned HTTP ~a\n~a" + (provider-name provider) status text))))) + +(def (openai-image-body provider path question detail) + (let ((body (make-hash-table)) + (msg (make-hash-table)) + (text-block (make-hash-table)) + (image-block (make-hash-table)) + (image-url (make-hash-table))) + (hash-put! body "model" (provider-model provider)) + (hash-put! body "max_tokens" (image-max-tokens)) + (hash-put! text-block "type" "text") + (hash-put! text-block "text" question) + (hash-put! image-url "url" (image-data-url path)) + (hash-put! image-url "detail" (openai-detail detail)) + (hash-put! image-block "type" "image_url") + (hash-put! image-block "image_url" image-url) + (hash-put! msg "role" "user") + (hash-put! msg "content" (list text-block image-block)) + (hash-put! body "messages" (list msg)) + body)) + +(def (anthropic-image-observe provider path question detail) + (let* ((url (string-append (provider-base-url provider) "/messages")) + (body (anthropic-image-body provider path question detail)) + (headers `(("Content-Type" . "application/json") + ("x-api-key" . ,(image-provider-key provider)) + ("anthropic-version" . "2023-06-01"))) + (body-json (json-object->string body))) + (let-values (((status text) (http-post-json url headers body-json))) + (if (= status 200) + (let ((answer (extract-anthropic-image-text (string->json-object text)))) + (format-image-observation provider path question detail answer)) + (format "Error: image_view provider ~a returned HTTP ~a\n~a" + (provider-name provider) status text))))) + +(def (anthropic-image-body provider path question detail) + (let ((body (make-hash-table)) + (msg (make-hash-table)) + (text-block (make-hash-table)) + (image-block (make-hash-table)) + (source (make-hash-table))) + (hash-put! body "model" (provider-model provider)) + (hash-put! body "max_tokens" (image-max-tokens)) + (hash-put! text-block "type" "text") + (hash-put! text-block "text" question) + (hash-put! source "type" "base64") + (hash-put! source "media_type" (image-mime-type path)) + (hash-put! source "data" (image-base64 path)) + (hash-put! image-block "type" "image") + (hash-put! image-block "source" source) + (hash-put! msg "role" "user") + (hash-put! msg "content" (list text-block image-block)) + (hash-put! body "messages" (list msg)) + body)) + +(def (google-image-observe provider path question detail) + (let* ((url (string-append (provider-base-url provider) + "/models/" (provider-model provider) + ":generateContent?key=" (image-provider-key provider))) + (body (google-image-body path question detail)) + (headers '(("Content-Type" . "application/json"))) + (body-json (json-object->string body))) + (let-values (((status text) (http-post-json url headers body-json))) + (if (= status 200) + (let ((answer (extract-google-image-text (string->json-object text)))) + (format-image-observation provider path question detail answer)) + (format "Error: image_view provider ~a returned HTTP ~a\n~a" + (provider-name provider) status text))))) + +(def (google-image-body path question detail) + (let ((body (make-hash-table)) + (content (make-hash-table)) + (text-part (make-hash-table)) + (image-part (make-hash-table)) + (inline-data (make-hash-table))) + (hash-put! text-part "text" question) + (hash-put! inline-data "mime_type" (image-mime-type path)) + (hash-put! inline-data "data" (image-base64 path)) + (hash-put! image-part "inline_data" inline-data) + (hash-put! content "role" "user") + (hash-put! content "parts" (list text-part image-part)) + (hash-put! body "contents" (list content)) + body)) + +(def (format-image-observation provider path question detail answer) + (if (or (not answer) (string=? (string-trim answer) "")) + (format "Error: image_view provider ~a returned no observation" + (provider-name provider)) + (format "Image: ~a\nBytes: ~a\nProvider: ~a\nModel: ~a\nDetail: ~a\nQuestion: ~a\n\nObservation:\n~a" + path + (safe-file-size path) + (provider-name provider) + (provider-model provider) + detail + question + (string-trim answer)))) + +(def (image-provider-key provider) + (or (config-get-provider-key (provider-name provider)) "")) + +(def (extract-openai-image-text json) + (let* ((choices (hash-ref json "choices" '())) + (choice (and (pair? choices) (car choices))) + (msg (and choice (hash-get choice "message"))) + (content (and msg (hash-get msg "content")))) + (json-content->text content))) + +(def (extract-anthropic-image-text json) + (json-content->text (hash-ref json "content" '()))) + +(def (extract-google-image-text json) + (let* ((candidates (hash-ref json "candidates" '())) + (candidate (and (pair? candidates) (car candidates))) + (content (and candidate (hash-get candidate "content"))) + (parts (and content (hash-ref content "parts" '())))) + (json-content->text parts))) + +(def (json-content->text content) + (cond + ((not content) "") + ((eq? content (void)) "") + ((string? content) content) + ((list? content) + (string-join + (filter string? + (map (lambda (block) + (cond + ((string? block) block) + ((and (hash-table? block) (hash-get block "text")) + (hash-get block "text")) + (else #f))) + content)) + "\n")) + (else ""))) + (def (run-image-backend path argv) (let* ((cwd (current-directory)) (allow-net? (config-bool "image_view" "allow_network" #f)) @@ -220,17 +404,60 @@ (put-string out new) (loop (+ start idx old-len))))))))) +(def (image-data-url path) + (string-append "data:" (image-mime-type path) ";base64," (image-base64 path))) + +(def (image-base64 path) + (std-base64-encode (read-file-bytevector path))) + +(def (read-file-bytevector path) + (let ((p (open-file-input-port path))) + (dynamic-wind + (lambda () (void)) + (lambda () + (let ((bv (get-bytevector-all p))) + (if (eof-object? bv) (make-bytevector 0) bv))) + (lambda () (close-port p))))) + +(def (image-mime-type path) + (let ((ext (normalized-extension path))) + (cond + ((or (equal? ext "jpg") (equal? ext "jpeg")) "image/jpeg") + ((equal? ext "png") "image/png") + ((equal? ext "gif") "image/gif") + ((equal? ext "webp") "image/webp") + ((equal? ext "bmp") "image/bmp") + ((or (equal? ext "tif") (equal? ext "tiff")) "image/tiff") + ((equal? ext "heic") "image/heic") + ((equal? ext "heif") "image/heif") + ((equal? ext "avif") "image/avif") + (else "application/octet-stream")))) + +(def (openai-detail detail) + (let ((d (string-downcase (or detail "")))) + (cond + ((or (equal? d "low") (equal? d "high") (equal? d "auto")) d) + (else "auto")))) + +(def (image-max-tokens) + (let ((v (config-ref "image_view" "max_tokens"))) + (if (and (number? v) (> v 0)) v 4096))) + (def (supported-image-extension? path) + (let ((ext (normalized-extension path))) + (and (member ext '("png" "jpg" "jpeg" "gif" "webp" "bmp" + "tif" "tiff" "heic" "heif" "avif")) + #t))) + +(def (normalized-extension path) (let ((ext0 (path-extension path))) - (and ext0 - (let* ((ext1 (string-downcase ext0)) - (ext (if (and (> (string-length ext1) 0) - (char=? (string-ref ext1 0) #\.)) - (substring ext1 1 (string-length ext1)) - ext1))) - (and (member ext '("png" "jpg" "jpeg" "gif" "webp" "bmp" - "tif" "tiff" "heic" "heif" "avif")) - #t))))) + (if ext0 + (let ((ext1 (string-downcase ext0))) + (if (and (> (string-length ext1) 0) + (char=? (string-ref ext1 0) #\.)) + (substring ext1 1 (string-length ext1)) + ext1)) + ""))) (def (image-too-large? path) (let ((size (safe-file-size path)) --- a/test/run.ss +++ b/test/run.ss @@ -466,11 +466,41 @@ (call-with-output-file tmp (lambda (out) (display "placeholder" out)) 'replace) - (parameterize ((*config* (make-hashtable equal-hash equal?))) - (let ([r (tool-execute "image_view" (args "path" tmp))]) - (check-pred! "image_view reports missing backend config" - r - (lambda (s) (str-contains? s "image_view is not configured")))))) + (let* ([srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [cfg (make-hashtable equal-hash equal?)] + [providers (make-hashtable equal-hash equal?)] + [openai (make-hashtable equal-hash equal?)] + [image-cfg (make-hashtable equal-hash equal?)] + [body "{\"choices\":[{\"message\":{\"content\":\"builtin vision ok\"}}]}"]) + (hashtable-set! openai "api_key" "unit-key") + (hashtable-set! openai "base_url" base-url) + (hashtable-set! providers "openai" openai) + (hashtable-set! image-cfg "provider" "openai") + (hashtable-set! image-cfg "model" "vision-unit") + (hashtable-set! cfg "providers" providers) + (hashtable-set! cfg "image_view" image-cfg) + (dynamic-wind + (lambda () (void)) + (lambda () + (serve-one-captured-json! srv captured 200 body) + (parameterize ((*config* cfg)) + (let* ([r (tool-execute "image_view" + (args "path" tmp "question" "focus"))] + [req (vector-ref captured 0)]) + (check-pred! "image_view built-in provider observation" + r + (lambda (s) + (and (str-contains? s "builtin vision ok") + (str-contains? s "Provider: openai") + (str-contains? s "Model: vision-unit")))) + (check-pred! "image_view built-in sends image_url data URL" + req + (lambda (s) + (and (str-contains? s "\"type\":\"image_url\"") + (str-contains? s "data:image/png;base64,"))))))) + (lambda () (tcp-close srv))))) ;; ── Bash tool tests ───────────────────────────────────────────────