updates
ober
052ecbb86fe466e6609939dc266b770f26c201d6
--- a/docs/escalation.md +++ b/docs/escalation.md @@ -140,6 +140,11 @@ verification or an expert model." Do not treat them as a factuality score. | `high-mean-entropy` | `mean_entropy` was above `max_mean_entropy`. | | `truncated-response` | The provider stopped because the reply hit the token cap. | +Lookup misses from repository exploration, such as `File not found`, `Path not +found`, and `Not a directory`, do not count toward `tool-error-streak`. They are +treated as recoverable path discovery rather than evidence that the model is +stuck. + The primary model can also explicitly request the expert by emitting: ```text --- a/src/jcode/core/escalation.ss +++ b/src/jcode/core/escalation.ss @@ -13,8 +13,9 @@ ;;; 2. no-text-rounds — last N assistant turns had no text content. ;;; Useful when a thinking model has stopped ;;; thinking and is reflex-emitting tool calls. -;;; 3. tool-error-streak — last N tool results in a row started with -;;; "error" / "Error". Model failing to recover. +;;; 3. tool-error-streak — last N non-benign tool results in a row +;;; started with "error" / "Error". Model +;;; failing to recover. ;;; 3b. no-progress — last N tool results, after canonicalization, ;;; collapse to the same value. Catches cases ;;; where the model is iterating without @@ -87,18 +88,26 @@ (def (tool-result-message? msg) (equal? (message-role msg) "tool")) +(def (benign-lookup-miss? trimmed) + ;; Repository exploration often includes a few guessed paths. Those are + ;; recoverable lookup misses, not a reason to spend expert budget. + (or (string-prefix? "Error: File not found:" trimmed) + (string-prefix? "Error: Path not found:" trimmed) + (string-prefix? "Error: Not a directory:" trimmed))) + (def (looks-like-error? text) (and text (string? text) (let ((trimmed (string-trim text))) - (or (string-prefix? "error" trimmed) - (string-prefix? "Error" trimmed) - (string-prefix? "ERROR" trimmed) - ;; MCP tool failures surface as "MCP Error: ..." — not caught by the - ;; prefixes above, so the tool-error-streak signal was blind to them. - (string-prefix? "MCP Error" trimmed) - ;; Common nested-result form from MCP wrapper - (string-contains trimmed "\"isError\":true"))))) + (and (not (benign-lookup-miss? trimmed)) + (or (string-prefix? "error" trimmed) + (string-prefix? "Error" trimmed) + (string-prefix? "ERROR" trimmed) + ;; MCP tool failures surface as "MCP Error: ..." — not caught by the + ;; prefixes above, so the tool-error-streak signal was blind to them. + (string-prefix? "MCP Error" trimmed) + ;; Common nested-result form from MCP wrapper + (string-contains trimmed "\"isError\":true")))))) (def (last-n-where pred lst n) ;; Walk lst from the END, collect at most n items where pred holds, --- a/test/run.ss +++ b/test/run.ss @@ -9,6 +9,7 @@ (jcode core message) (jcode core grok-auth) (jcode core config) + (jcode core escalation) (jcode core agent) (jcode core models) (jcode core secrets-import) @@ -1676,6 +1677,25 @@ (forge-mark-no-progress-nudged!) (check! "agent breaker nudge marked used" (forge-no-progress-nudge-used?) #t)) +(section "=== expert escalation ===") +;; Guessed paths during repository exploration are normal recovery steps for a +;; local model. They should not consume expert budget as a tool-error streak, +;; while real tool failures still should. +(let* ([msgs (list (make-tool-result "r1" "Error: File not found: /repo/src/jcode/agent.ss") + (make-tool-result "r2" "Error: Path not found: /repo/bin") + (make-tool-result "r3" "Error: Not a directory: /repo/README.md"))] + [resp (make-assistant-message "I'll keep looking.")]) + (check! "escalation ignores lookup misses" + (should-escalate? msgs resp '()) #f)) + +(let* ([msgs (list (make-tool-result "r1" "Error: command failed: exit 1") + (make-tool-result "r2" "Error: sandbox denied") + (make-tool-result "r3" "MCP Error: server disconnected"))] + [resp (make-assistant-message "I'll keep looking.")] + [reason (should-escalate? msgs resp '())]) + (check! "escalation keeps real tool-error streak" + (and reason (cdr (assq 'signal reason))) 'tool-error-streak)) + (section "=== verified-run: coding workflow on a REAL file + REAL shell verify ===") ;; The verify-gate/best-of-k tests above use mock callables. This one drives the ;; live-model bridge's genuinely new code — do-edit (real write-file-string),