tool: replace SearXNG with vendored in-process jerbsearch
ober
fbdeba200812a90c8d8ae315fe790379f39c98ef
--- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "vendor/termbox2"] path = vendor/termbox2 url = https://github.com/termbox/termbox2.git +[submodule "vendor/jerboa-websearch"] + path = vendor/jerboa-websearch + url = git@github.com:ober/jerboa-websearch.git --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SCHEME = scheme JERBOA_HOME ?= $(HOME)/mine/jerboa -LIBDIRS = --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src +LIBDIRS = --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src:vendor/jerboa-websearch/src JERBUILD = $(SCHEME) --libdirs $(JERBOA_HOME)/lib --script $(JERBOA_HOME)/jerbuild.ss # Library paths for FFI shared objects (macOS: dylib, Linux: so) @@ -110,7 +110,7 @@ test: build binary: build JERBOA_HOME=$(JERBOA_HOME) \ DYLD_LIBRARY_PATH=$(LDPATH) LD_LIBRARY_PATH=$(LDPATH) \ - $(SCHEME) -q --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src \ + $(SCHEME) -q --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src:vendor/jerboa-websearch/src \ --script build-binary.ss cp vendor/termbox2/jcode_tui_shim.dylib ./jcode_tui_shim.dylib 2>/dev/null || true cp vendor/termbox2/jcode_tui_shim.so ./jcode_tui_shim.so 2>/dev/null || true @@ -166,7 +166,7 @@ jcode-musl: linux-local freebsd: build JERBOA_HOME=$(JERBOA_HOME) \ - $(SCHEME) -q --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src \ + $(SCHEME) -q --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src:vendor/jerboa-websearch/src \ --script build-binary.ss clean: --- a/build-binary.ss +++ b/build-binary.ss @@ -97,6 +97,7 @@ (format "~a/lib" jerboa-dir))) (list (cons "lib" "lib")) (list (cons "vendor/chez-sqlite/src" "vendor/chez-sqlite/src")) + (list (cons "vendor/jerboa-websearch/src" "vendor/jerboa-websearch/src")) (list (cons "." ".")) (library-directories))) @@ -315,16 +316,36 @@ "std/os/sandbox" "std/text/json" "std/text/glob" + "std/text/html" + "std/text/html-parse" + "std/text/yaml/nodes" + "std/text/yaml/reader" + "std/text/yaml/writer" + "std/text/yaml" "std/net/tcp" "std/net/tls-rustls" "std/net/request" "std/net/uri" + "std/net/http" "std/db/sqlite"))) +;; jerbsearch (vendored) — used by jcode/tool/web for in-process metasearch. +;; Compiled .so files live under vendor/jerboa-websearch/src/jerbsearch/. +(define jerbsearch-modules + (map (lambda (m) (format "vendor/jerboa-websearch/src/jerbsearch/~a.so" m)) + '("engine" + "query" + "result" + "settings" + "metrics" + "search" + "engines/duckduckgo"))) + (apply make-boot-file "jcode.boot" '("scheme" "petite") (append (existing-so-files external-libs) (list "vendor/chez-sqlite/src/chez-sqlite.so") + (existing-so-files jerbsearch-modules) (existing-so-files (map (lambda (m) (format "~a.so" m)) jcode-modules)))) --- a/build-jcode-musl.sh +++ b/build-jcode-musl.sh @@ -47,7 +47,7 @@ echo "[2/2] Running musl build..." LD_LIBRARY_PATH="${SCRIPT_DIR}/vendor/chez-sqlite:${SCRIPT_DIR}/vendor/termbox2:${JERBOA_LIB}:." \ JERBOA_HOME="${JERBOA_HOME}" \ -scheme -q --libdirs "${JERBOA_LIB}:./lib:vendor/chez-sqlite/src" \ +scheme -q --libdirs "${JERBOA_LIB}:./lib:vendor/chez-sqlite/src:vendor/jerboa-websearch/src" \ <build-jcode-musl.ss # Verify --- a/src/jcode/tool/web.ss +++ b/src/jcode/tool/web.ss @@ -1,4 +1,4 @@ -;;; jcode web tool — HTTP/HTTPS fetch + SearXNG search +;;; jcode web tool — HTTP/HTTPS fetch + in-process metasearch (jerbsearch) (export init-web-tools) @@ -6,18 +6,29 @@ :std/net/uri :std/text/json :std/misc/string + :jerbsearch/search + :jerbsearch/query + :jerbsearch/result + :jerbsearch/engines/duckduckgo :jcode/core/log :jcode/tool/registry) (def logger (make-logger "tool.web")) +(def *engines-registered?* #f) + +(def (ensure-engines!) + (unless *engines-registered?* + (register!) + (set! *engines-registered?* #t))) + (def (init-web-tools) (register-tool! "fetch" "Fetch a URL via HTTP/HTTPS. Supports GET and POST. Returns HTTP status and response body." (make-fetch-schema) handle-fetch) (register-tool! "web_search" - "Search the web via a self-hosted SearXNG instance (override URL with SEARX_URL env, default http://127.0.0.1:8888). Returns a ranked list of {title, url, snippet}. Use this when you need fresh information or do not know the URL; follow up with fetch on a specific result URL." + "Search the web via the in-process jerbsearch metasearch engine (DuckDuckGo HTML backend; no external service required). Returns a ranked list of {title, url, snippet}. Use this when you need fresh information or do not know the URL; follow up with fetch on a specific result URL." (make-search-schema) handle-search)) @@ -76,7 +87,7 @@ (request-close resp) (format "Status: ~a\n~a" status text))) -;; ---- web_search (self-hosted SearXNG, JSON API) ---- +;; ---- web_search (in-process jerbsearch) ---- (def (make-search-schema) (let ((schema (make-hash-table)) @@ -105,36 +116,17 @@ (unless query (error 'web_search "Missing required parameter: query")) (log-info logger "web_search" `((q . ,query) (count . ,count))) (try - (searxng-search query count) + (jerbsearch-do query count) (catch (e) (format "Error: ~a" (err->string e)))))) -(def (searx-base-url) - (let ((env (getenv "SEARX_URL"))) - (if (and env (> (string-length env) 0)) - env - "http://127.0.0.1:8888"))) - -(def (searxng-search query count) - (let* ((base (searx-base-url)) - (url (string-append base "/search?q=" (uri-encode query) - "&format=json&categories=general")) - (headers '(("User-Agent" . "jcode/web_search") - ("Accept" . "application/json"))) - (resp (http-get url headers #f)) - (status (request-status resp)) - (text (request-text resp))) - (request-close resp) - (cond - ((not (= status 200)) - (format "SearXNG HTTP ~a (is ~a running? set SEARX_URL to override)\n~a" - status base text)) - (#t - (let* ((j (string->json-object text)) - (rs (or (hash-ref j "results" #f) '())) - (limit (min count (length rs))) - (taken (take-up-to rs limit))) - (format-search-results query taken)))))) +(def (jerbsearch-do query count) + (ensure-engines!) + (let* ((rq (parse-raw-query query)) + (rc (search-execute '() rq)) + (results (rc-get-ordered-results rc (lambda (eng) 1.0))) + (taken (take-up-to results count))) + (format-search-results query taken))) (def (take-up-to xs n) (cond @@ -150,14 +142,14 @@ (cond ((null? rs) (string-append - (format "SearXNG: ~a (~a results)\n\n" + (format "jerbsearch: ~a (~a results)\n\n" query (length results)) (string-join (reverse acc) "\n\n"))) (#t (let* ((r (car rs)) - (title (or (hash-ref r "title" "") "")) - (url (or (hash-ref r "url" "") "")) - (snip (truncate-snippet (or (hash-ref r "content" "") "")))) + (title (or (main-result-title r) "")) + (url (or (main-result-url r) "")) + (snip (truncate-snippet (or (main-result-content r) "")))) (loop (cdr rs) (+ i 1) (cons (format "~a. ~a\n ~a\n ~a" i title url snip) acc))))))))) new file mode 160000 --- /dev/null +++ b/vendor/jerboa-websearch @@ -0,0 +1 @@ +Subproject commit fb0643d54081717d2552ffec76ea5fdb89e475fc