expert: harden escalation + add /expert slash command
ober
6348555462e3757a999b9c8fe78350c00f2cb89e
--- a/src/jcode/core/escalation.ss +++ b/src/jcode/core/escalation.ss @@ -49,13 +49,18 @@ (def (escalation-config key) "Look up an escalation knob: first under expert.escalation in jcode.json, - then fall back to *escalation-defaults*." + then fall back to *escalation-defaults*. Setting a key to false in + config disables it; absent keys use the default. We descend manually + to the leaf block and probe with hash-key? — config-ref alone can't + distinguish an explicit JSON false from an absent key." (let* ((str-key (symbol->string key)) - (configured (config-ref "expert" "escalation" str-key))) - (if (eq? configured #f) - (let ((pair (assq key *escalation-defaults*))) - (and pair (cdr pair))) - configured))) + (block (config-ref "expert" "escalation"))) + (cond + ((and block (hash-table? block) (hash-key? block str-key)) + (hash-get block str-key)) + (else + (let ((pair (assq key *escalation-defaults*))) + (and pair (cdr pair))))))) ;;; --- pure detection helpers (no I/O) --- @@ -119,19 +124,28 @@ (tool-names . ,(map car current-sig)))))))) (def (detect-no-text-rounds messages response) + ;; "Empty" here means an assistant turn with no text AND no tool calls. + ;; Tool-only turns are normal in a coding agent (read/grep/edit chains) + ;; so they must NOT count toward this signal — otherwise every healthy + ;; multi-step task auto-escalates. Repeated identical tool calls are + ;; caught by detect-identical-loop instead. (let ((threshold (escalation-config 'max_rounds_without_text))) (if (not threshold) #f - (let* ((no-text? (lambda (m) - (and (equal? (message-role m) "assistant") - (let ((c (message-content m))) - (or (not c) (string=? (string-trim (or c "")) "")))))) - (current-no-text? (no-text? response)) + (let* ((empty-asst? + (lambda (m) + (and (equal? (message-role m) "assistant") + (let ((tcs (message-tool-calls m)) + (c (message-content m))) + (and (or (not tcs) (null? tcs)) + (or (not c) + (string=? (string-trim (or c "")) ""))))))) + (current-empty? (empty-asst? response)) (prior-asst (filter (lambda (m) (equal? (message-role m) "assistant")) messages)) - (recent (last-n-where no-text? prior-asst (- threshold 1))) - (run-length (+ (if current-no-text? 1 0) (length recent)))) - (and current-no-text? + (recent (last-n-where empty-asst? prior-asst (- threshold 1))) + (run-length (+ (if current-empty? 1 0) (length recent)))) + (and current-empty? (>= run-length threshold) `((signal . no-text-rounds) (run-length . ,run-length) --- a/src/jcode/core/expert.ss +++ b/src/jcode/core/expert.ss @@ -93,17 +93,42 @@ (config-ref "expert" "model")) "")) +;; When escalating, rebuild the leading system message to include a brief +;; note that the expert is being consulted and why. Anthropic collapses +;; system messages into a single body field; OpenAI/others take one at the +;; head; appending to the existing system prompt works uniformly. Returns a +;; NEW list — doesn't mutate. +(def (with-expert-handoff messages reason-text) + (let ((handoff-note + (format "\n\n[EXPERT HANDOFF] You are being consulted as the expert model because the primary model appeared stuck or low-confidence (~a). The conversation above is the full context. Provide the best answer you can for the user's most recent request — be concrete and direct." + reason-text))) + (cond + ((null? messages) messages) + ((equal? (message-role (car messages)) "system") + (let* ((sys (car messages)) + (orig (or (message-content sys) ""))) + (cons (make-system-message (string-append orig handoff-note)) + (cdr messages)))) + (else + (cons (make-system-message (string-trim handoff-note)) messages))))) + ;; Non-streaming wrapper. Calls the primary provider; if its response ;; contains the sentinel OR triggers any auto-escalation signal, and an ;; expert is configured, re-sends the same messages to the expert and ;; returns that response instead. The primary's attempt is discarded. +;; If the expert call itself errors (bad key, timeout, …) we fall back +;; to the primary's attempt rather than dropping a working response on +;; the floor. (def (chat-with-expert provider messages tools) (let-values (((response stats) (provider-chat-with-stats provider messages tools))) (let* ((content (message-content response)) (sentinel? (wants-expert? content)) (auto-reason (should-escalate? messages response stats)) - (escalate? (or sentinel? auto-reason))) + (escalate? (or sentinel? auto-reason)) + (reason-text (if auto-reason + (format-escalation-reason auto-reason) + "sentinel"))) (cond ((and escalate? (config-expert-enabled?)) (let ((expert (get-expert-provider))) @@ -112,10 +137,18 @@ (to . ,(provider-name expert)) (model . ,(provider-model expert)) (trigger . ,(if sentinel? 'sentinel 'auto)) - (reason . ,(if auto-reason - (format-escalation-reason auto-reason) - "sentinel")))) - (provider-chat expert messages tools))) + (reason . ,reason-text))) + (guard (e [#t + (log-warn logger "expert-call-failed" + `((err . ,(err->string e)))) + (if sentinel? + (make-assistant-message + (strip-expert-sentinel content) + (message-tool-calls response)) + response)]) + (provider-chat expert + (with-expert-handoff messages reason-text) + tools)))) (sentinel? (log-warn logger "expert-requested-but-not-configured" '()) (make-assistant-message @@ -130,6 +163,11 @@ ;; low confidence, truncation …). On any trigger, stream the expert ;; response in place of the primary's. The expert's (content tool-calls ;; usage) is what gets returned and recorded. +;; +;; If the expert call errors (auth, timeout, partial stream death) we push +;; a brief notice through token-cb and fall back to the primary's content, +;; tool-calls and usage — the user already saw the primary tokens, so this +;; keeps the session coherent rather than dropping a working attempt. (def (stream-chat-with-expert provider messages tools token-cb) (let-values (((content tcs usage stats) (provider-stream-chat-with-stats provider messages tools token-cb))) @@ -138,7 +176,10 @@ ;; inspect tool-call shapes without touching the live session. (synth (make-assistant-message content tcs)) (auto-reason (should-escalate? messages synth stats)) - (escalate? (or sentinel? auto-reason))) + (escalate? (or sentinel? auto-reason)) + (reason-text (if auto-reason + (format-escalation-reason auto-reason) + "sentinel"))) (cond ((and escalate? (config-expert-enabled?)) (let ((expert (get-expert-provider))) @@ -147,18 +188,27 @@ (to . ,(provider-name expert)) (model . ,(provider-model expert)) (trigger . ,(if sentinel? 'sentinel 'auto)) - (reason . ,(if auto-reason - (format-escalation-reason auto-reason) - "sentinel")))) + (reason . ,reason-text))) (when token-cb (token-cb (format "\n\n[escalating to ~a/~a~a]\n\n" (provider-name expert) (provider-model expert) (if auto-reason - (string-append " — " (format-escalation-reason auto-reason)) + (string-append " — " reason-text) "")))) - (provider-stream-chat expert messages tools token-cb))) + (guard (e [#t + (log-warn logger "expert-call-failed" + `((err . ,(err->string e)))) + (when token-cb + (token-cb + (format "\n[expert call failed: ~a — keeping primary response]\n" + (err->string e)))) + (values (if sentinel? (strip-expert-sentinel content) content) + tcs usage)]) + (provider-stream-chat expert + (with-expert-handoff messages reason-text) + tools token-cb)))) (sentinel? (log-warn logger "expert-requested-but-not-configured" '()) (values (strip-expert-sentinel content) tcs usage)) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -1089,6 +1089,7 @@ (tu-table (make-hash-table)) (current-idx (make-parameter #f)) (usage-acc (make-hash-table)) + (finish-reason-box (box #f)) (event-type #f)) (when (tracing?) (log-trace logger "anthropic-stream-request" @@ -1159,6 +1160,15 @@ (when (and usage (hash-table? usage)) (hash-for-each (lambda (k v) (hash-put! usage-acc k v)) usage)))))) ((equal? event-type "message_delta") + ;; delta.stop_reason: "end_turn" | "max_tokens" | + ;; "stop_sequence" | "tool_use". We map max_tokens to + ;; the OpenAI-style "length" so detect-truncated fires. + (let ((delta (hash-get json "delta"))) + (when (and delta (hash-table? delta)) + (let ((sr (hash-get delta "stop_reason"))) + (when (and sr (string? sr)) + (set-box! finish-reason-box + (if (equal? sr "max_tokens") "length" sr)))))) (let ((usage (hash-get json "usage"))) (when (and usage (hash-table? usage)) (hash-for-each (lambda (k v) (hash-put! usage-acc k v)) usage)))) @@ -1189,8 +1199,9 @@ (cons 'tokens-out (or (hash-get usage-acc "output_tokens") 0)) (cons 'cost (compute-cost (provider-model provider) usage-acc))) - ;; Anthropic streaming does not surface logprobs; stats are empty. - (build-stats #f '() '()))))) + ;; Anthropic streaming does not surface logprobs, but stop_reason + ;; is captured above so detect-truncated still works. + (build-stats (unbox finish-reason-box) '() '()))))) ;; provider-stream-chat-with-stats: like provider-stream-chat but also ;; returns a `stats` alist with finish_reason / mean_logprob / mean_entropy --- a/src/jcode/ui/tui.ss +++ b/src/jcode/ui/tui.ss @@ -21,6 +21,7 @@ :jcode/core/config :jcode/core/models :jcode/core/agent + :jcode/core/expert :jcode/core/debug-repl :jcode/core/skill :jcode/core/builtin-skills @@ -496,6 +497,7 @@ " /ask-gemini Second opinion from gemini CLI (sandboxed)" " /ask-codex Second opinion from codex CLI (sandboxed)" " /ask-opencode Second opinion from opencode CLI (sandboxed)" + " /expert <q> Force this prompt to the configured expert model" " /theme Cycle theme (Ctrl-T)" " /sidebar Toggle sidebar (Ctrl-B)" " /quit Exit" @@ -564,6 +566,11 @@ (equal? cmd "ask-opencode")) (handle-ask-external! state (string->symbol (substring cmd 4 (string-length cmd))))) + ((equal? cmd "expert") + (handle-expert! state "")) + ((string-prefix? "expert " cmd) + (handle-expert! state + (substring cmd 7 (string-length cmd)))) ((equal? cmd "sessions") (let ((sessions (session-list))) (add-message! state @@ -1041,6 +1048,40 @@ (send-worker-event! gen (list 'ask-result provider result)))))))) +;; ---- /expert <prompt> ---- +;; Force the next turn to the configured expert by parameterizing the +;; provider/model overrides for the duration of run-agent!. The worker +;; captures those parameters at spawn time, so when this dynamic extent +;; exits the overrides naturally revert for subsequent turns. +(def (handle-expert! state arg) + (let ((prompt (string-trim arg))) + (cond + ((not (config-expert-enabled?)) + (add-message! state + (msg-block-system + "No expert configured. Add an \"expert\" block to jcode.json:\n { \"expert\": { \"provider\": \"openrouter\", \"model\": \"deepseek/deepseek-chat\" } }"))) + ((app-state-agent-busy? state) + (add-message! state + (msg-block-system + "Wait for the current response to finish, then re-run /expert."))) + ((string=? prompt "") + (add-message! state + (msg-block-system + (format "Usage: /expert <prompt>\nForces this prompt to ~a/~a (the configured expert)." + (config-ref "expert" "provider") + (config-ref "expert" "model"))))) + (else + (let ((expert-provider (config-ref "expert" "provider")) + (expert-model (config-ref "expert" "model"))) + (add-message! state (msg-block-user (string-append "/expert " prompt))) + (add-message! state + (msg-block-system + (format "Routing to expert: ~a/~a" expert-provider expert-model))) + (app-state-scroll-offset-set! state 0) + (parameterize ((current-provider-override expert-provider) + (current-model-override expert-model)) + (run-agent! state prompt))))))) + ;; ---- /compact ---- ;; Summarize older user/assistant turns via the current LLM, then replace ;; both app-state-messages and the session DB with [summary + last K turns].