test: live say-hello smoke test per configured provider
ober
696c53e8acded3b20ad514007a98d88c2d5cad6b
--- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ TUI_SHIM_DIR := $(CURDIR)/vendor/termbox2 NATIVE_LIB_DIR := $(JERBOA_HOME)/lib LDPATH := $(SHIM_DIR):$(TUI_SHIM_DIR):$(SQLITE_LIB_DIR):$(NATIVE_LIB_DIR) -.PHONY: all help build gen run test clean repl binary install tui-shim run-tui jcode-musl linux linux-check linux-local docker purge-stale sqlite-shim android android-clean +.PHONY: all help build gen run test test-providers clean repl binary install tui-shim run-tui jcode-musl linux linux-check linux-local docker purge-stale sqlite-shim android android-clean all: help @@ -18,6 +18,7 @@ help: @echo "Available targets:" @echo " build Compile src/ → lib/ and pre-compile imports" @echo " test Run test suite (test/run.ss)" + @echo " test-providers Live 'say hello' smoke test per configured provider" @echo " run Start interactive agent REPL" @echo " run-tui Start TUI mode" @echo " repl Open a bare Scheme REPL with project libdirs" @@ -105,6 +106,13 @@ test: build DYLD_LIBRARY_PATH=$(LDPATH) LD_LIBRARY_PATH=$(LDPATH) \ $(SCHEME) $(LIBDIRS) --script test/run.ss +# Live smoke test — one round-trip per provider with a configured key. +# Providers without a key are SKIPped, not failed. Excluded from `test` +# because it makes real network calls. +test-providers: build + DYLD_LIBRARY_PATH=$(LDPATH) LD_LIBRARY_PATH=$(LDPATH) \ + $(SCHEME) $(LIBDIRS) --script test/run-providers.ss + # --- Static binary targets --- binary: build new file mode 100644 --- /dev/null +++ b/test/run-providers.ss @@ -0,0 +1,84 @@ +#!/usr/bin/env scheme --script +;;; Live provider smoke test — one "say hello" round-trip per provider +;;; with a configured API key. Providers without a key are SKIPped, not failed. + +(import (chezscheme) + (jcode core log) + (jcode core config) + (jcode core message) + (jcode provider provider)) + +;; ── Helpers ────────────────────────────────────────────────────── + +(define pass-count 0) +(define fail-count 0) +(define skip-count 0) + +(define (truncate-string s n) + (if (> (string-length s) n) + (string-append (substring s 0 n) "...") + s)) + +(define (condition->string e) + (with-output-to-string (lambda () (display-condition e)))) + +(define (test-provider provider-name model) + (let ((key (config-get-provider-key provider-name))) + (cond + ((not key) + (set! skip-count (+ skip-count 1)) + (printf " SKIP: ~a (~a) — no API key~n" provider-name model)) + (else + (printf " ... ~a (~a)~n" provider-name model) + (guard + (e [#t + (set! fail-count (+ fail-count 1)) + (printf " FAIL: ~a — ~a~n" provider-name + (condition->string e))]) + (let* ((p (make-provider provider-name key model)) + (msg (make-user-message "Say hello in exactly one short sentence.")) + (r (provider-chat p (list msg) '()))) + (cond + ((not (message? r)) + (set! fail-count (+ fail-count 1)) + (printf " FAIL: ~a — not a message: ~s~n" provider-name r)) + ((not (message-content r)) + (set! fail-count (+ fail-count 1)) + (printf " FAIL: ~a — empty content~n" provider-name)) + ((zero? (string-length (message-content r))) + (set! fail-count (+ fail-count 1)) + (printf " FAIL: ~a — zero-length content~n" provider-name)) + (else + (set! pass-count (+ pass-count 1)) + (printf " PASS: ~a → ~s~n" provider-name + (truncate-string (message-content r) 80)))))))))) + +;; ── Setup ───────────────────────────────────────────────────────── + +(current-log-level 'warn) +(load-config) + +;; ── Providers ───────────────────────────────────────────────────── +;; One cheap+fast model per provider. The openrouter pick is qwen3-30b +;; specifically — qwen3 on openrouter is what surfaced the make-message +;; arity bug fixed in 56ab041, so it doubles as a regression test. + +(define providers + '(("anthropic" . "claude-haiku-35-20241022") + ("openai" . "gpt-4o-mini") + ("openrouter" . "qwen/qwen3-30b-a3b") + ("deepseek" . "deepseek-chat") + ("google" . "gemini-2.5-flash") + ("xai" . "grok-3-mini") + ("groq" . "llama-3.1-8b-instant") + ("mistral" . "mistral-small-latest") + ("together" . "meta-llama/Llama-3.3-70B-Instruct-Turbo") + ("cerebras" . "llama-3.1-8b") + ("perplexity" . "sonar"))) + +(printf "~n=== live providers (say hello) ===~n") +(for-each (lambda (entry) (test-provider (car entry) (cdr entry))) providers) + +(printf "~n~a passed, ~a failed, ~a skipped~n" + pass-count fail-count skip-count) +(when (> fail-count 0) (exit 1))