tool: replace Brave web_search with self-hosted SearXNG
ober
8e09be0db841978167163cc848dd14bfb983c862
--- a/build-binary.ss +++ b/build-binary.ss @@ -316,6 +316,7 @@ "std/net/tcp" "std/net/tls-rustls" "std/net/request" + "std/net/uri" "std/db/sqlite"))) (apply make-boot-file "jcode.boot" '("scheme" "petite") --- a/src/jcode/tool/web.ss +++ b/src/jcode/tool/web.ss @@ -1,12 +1,12 @@ -;;; jcode web tool — HTTP/HTTPS fetch +;;; jcode web tool — HTTP/HTTPS fetch + SearXNG search (export init-web-tools) (import :std/net/request + :std/net/uri :std/text/json :std/misc/string :jcode/core/log - :jcode/core/config :jcode/tool/registry) (def logger (make-logger "tool.web")) @@ -17,7 +17,7 @@ (make-fetch-schema) handle-fetch) (register-tool! "web_search" - "Search the web via Brave Search. Returns a ranked list of {title, url, snippet}. Use this when you need fresh information or do not know the URL; follow up with fetch on a specific result URL." + "Search the web via a self-hosted SearXNG instance (override URL with SEARX_URL env, default http://127.0.0.1:8888). Returns a ranked list of {title, url, snippet}. Use this when you need fresh information or do not know the URL; follow up with fetch on a specific result URL." (make-search-schema) handle-search)) @@ -76,7 +76,7 @@ (request-close resp) (format "Status: ~a\n~a" status text))) -;; ---- web_search (Brave Search API) ---- +;; ---- web_search (self-hosted SearXNG, JSON API) ---- (def (make-search-schema) (let ((schema (make-hash-table)) @@ -89,16 +89,11 @@ (hash-put! props "query" q-p)) (let ((c-p (make-hash-table))) (hash-put! c-p "type" "integer") - (hash-put! c-p "description" "Number of results to return (1-20, default 5).") + (hash-put! c-p "description" "Maximum results to return (1-20, default 5).") (hash-put! props "count" c-p)) (hash-put! schema "properties" props) schema)) -(def (brave-api-key) - (or (getenv "BRAVE_API_KEY") - (config-ref "brave_api_key") - (config-ref "providers" "brave" "api_key"))) - (def (handle-search args) (let* ((query (hash-ref args "query" #f)) (count (let ((c (hash-ref args "count" #f))) @@ -106,24 +101,25 @@ ((or (not c) (eq? c (void))) 5) ((number? c) (max 1 (min 20 (exact c)))) ((string? c) (max 1 (min 20 (string->number c)))) - (else 5)))) - (key (brave-api-key))) + (#t 5))))) (unless query (error 'web_search "Missing required parameter: query")) - (unless key - (error 'web_search - "No Brave Search API key. Set BRAVE_API_KEY env var or add 'brave_api_key' to jcode.json.")) (log-info logger "web_search" `((q . ,query) (count . ,count))) (try - (brave-search query count key) + (searxng-search query count) (catch (e) (format "Error: ~a" (err->string e)))))) -(def (brave-search query count key) - (let* ((url (string-append - "https://api.search.brave.com/res/v1/web/search?q=" - (url-encode query) - "&count=" (number->string count))) - (headers `(("X-Subscription-Token" . ,key) +(def (searx-base-url) + (let ((env (getenv "SEARX_URL"))) + (if (and env (> (string-length env) 0)) + env + "http://127.0.0.1:8888"))) + +(def (searxng-search query count) + (let* ((base (searx-base-url)) + (url (string-append base "/search?q=" (uri-encode query) + "&format=json&categories=general")) + (headers '(("User-Agent" . "jcode/web_search") ("Accept" . "application/json"))) (resp (http-get url headers #f)) (status (request-status resp)) @@ -131,42 +127,43 @@ (request-close resp) (cond ((not (= status 200)) - (format "Brave Search HTTP ~a\n~a" status text)) - (else (format-brave-results query text))))) + (format "SearXNG HTTP ~a (is ~a running? set SEARX_URL to override)\n~a" + status base text)) + (#t + (let* ((j (string->json-object text)) + (rs (or (hash-ref j "results" #f) '())) + (limit (min count (length rs))) + (taken (take-up-to rs limit))) + (format-search-results query taken)))))) -(def (format-brave-results query body) - (let* ((root (string->json-object body)) - (web (and (hash-table? root) (hash-get root "web"))) - (results (and (hash-table? web) (hash-get web "results")))) - (cond - ((or (not results) (null? results)) - (format "No results for: ~a" query)) - (else - (let loop ((rs results) (i 1) (acc '())) - (cond - ((null? rs) - (string-append - (format "Brave Search: ~a (~a results)\n\n" - query (length results)) - (string-join (reverse acc) "\n\n"))) - (else - (let* ((r (car rs)) - (title (or (hash-get r "title") "")) - (url (or (hash-get r "url") "")) - (snip (or (hash-get r "description") ""))) - (loop (cdr rs) (+ i 1) - (cons (format "~a. ~a\n ~a\n ~a" - i title url (strip-tags snip)) - acc)))))))))) +(def (take-up-to xs n) + (cond + ((or (null? xs) (<= n 0)) '()) + (#t (cons (car xs) (take-up-to (cdr xs) (- n 1)))))) -(def (strip-tags s) - ;; Brave snippets contain <strong>...</strong> markup; flatten for plain text. - (let loop ((chars (string->list s)) (out '()) (in-tag? #f)) - (cond - ((null? chars) (list->string (reverse out))) - ((and (not in-tag?) (char=? (car chars) #\<)) - (loop (cdr chars) out #t)) - ((and in-tag? (char=? (car chars) #\>)) - (loop (cdr chars) out #f)) - (in-tag? (loop (cdr chars) out #t)) - (else (loop (cdr chars) (cons (car chars) out) #f))))) +(def (format-search-results query results) + (cond + ((null? results) + (format "No results for: ~a" query)) + (#t + (let loop ((rs results) (i 1) (acc '())) + (cond + ((null? rs) + (string-append + (format "SearXNG: ~a (~a results)\n\n" + query (length results)) + (string-join (reverse acc) "\n\n"))) + (#t + (let* ((r (car rs)) + (title (or (hash-ref r "title" "") "")) + (url (or (hash-ref r "url" "") "")) + (snip (truncate-snippet (or (hash-ref r "content" "") "")))) + (loop (cdr rs) (+ i 1) + (cons (format "~a. ~a\n ~a\n ~a" i title url snip) + acc))))))))) + +(def (truncate-snippet s) + (let ((len (string-length s))) + (if (> len 240) + (string-append (substring s 0 240) "...") + s)))