Keep verified provider runs alive
ober
80aad4af79915433cb0ee2b8241477e9b06d260f
--- a/docs/rle-benchmark-optimization.md +++ b/docs/rle-benchmark-optimization.md @@ -1179,3 +1179,47 @@ worker gates. Its Rust dependency step stops on the pre-existing vendored lock: transitive crate `spin` 0.9.8 is yanked. That dependency is outside this benchmark patch; the residual is reported rather than hidden or addressed with an unrelated lockfile update. + +### Restored-Credit Cohort And Transport Follow-Up + +OpenRouter credits were restored and the task-identical four-model cohort ran +with expert mode, 72 turns, and 1,800 seconds per model. GLM and Qwen passed all +22 public and six hidden probes. DeepSeek passed public tests and five hidden +probes, still accepting illegal `2!`. Kimi terminated on an HTTP silence +watchdog before writing a file. The resulting cohort improved from 2/4 to 3/4 +public passes and from 1/4 to 2/4 full-spec passes. + +| Primary | Public | Hidden | Cost | +|---|---:|---:|---:| +| GLM 5.2 | 22/22 | 6/6 | $0.2115 | +| DeepSeek V4 Pro | 22/22 | 5/6 | $0.1383 | +| Qwen 3.7 Plus | 22/22 | 6/6 | $0.2440 | +| Kimi K2.7 Code | 0/22 | 0/6 | $0.0581 | + +The Kimi error used `HTTP read timed out`, while the retry policy recognized +only `stream read timed out`. Jcode now treats both no-response watchdog forms +as retryable. Verified provider calls also use the streaming transport and +collect tokens at the workflow seam, allowing OpenRouter keepalives and SSE +progress to prevent the same 600-second false-death condition. Ordinary chat +behavior is unchanged. Streaming OpenAI-compatible usage now goes through the +shared usage parser so cache-write and reasoning fields are no longer dropped. + +Deterministic coverage checks the timeout policy, terminal ordinary errors, +the verified `stream: true` request, response collection, and all usage fields. +The final full suite passes 1,179 tests, zero failures, and one environment +skip; ten newly present `/doit` completion checks account for the increase from +the first transport-validation run. + +The normal 32K Kimi rerun stayed productive for the entire 30-minute window, +completed 25 Kimi and eight DeepSeek expert calls, reached verification, and +left a 290-line candidate. It scored 14/22 public; its mechanical 2/6 hidden +score came only from rejection probes that failed for the wrong reason. Usage +was 653,105 input, 99,568 output, 294,592 cache-read, 11,674 reasoning tokens, +and $0.6663. This confirms the transport fix but not model success. + +A Kimi-only 16K default was then tested because one primary turn consumed the +entire 32,768-token allowance. The cap reduced that runaway but the full run +ended with no promoted file and zero public positive cases after 39 provider +calls, costing $0.5718. Together with the earlier unsuccessful 8K experiment, +this is negative evidence for retaining a model-specific cap. The cap was +removed; explicit `JCODE_MAX_TOKENS` and configured limits remain available. --- a/src/jcode/core/expert.ss +++ b/src/jcode/core/expert.ss @@ -30,6 +30,7 @@ expert-prompt-instructions expert-handoff-instruction chat-with-expert + chat-with-expert-via-stream stream-chat-with-expert current-expert-cb) @@ -310,3 +311,14 @@ (log-warn logger "expert-requested-but-not-configured" '()) (values (strip-expert-sentinel content) tcs usage)) (else (values content tcs usage)))))) + +(def (chat-with-expert-via-stream provider messages tools) + ;; Verified workflows do not display incremental model text, but using the + ;; streaming transport still lets gateways send progress bytes during long + ;; generations. Collect the response into the same message shape expected by + ;; the workflow backend and record the usage returned by the wrapper. + (let-values (((content tool-calls usage) + (stream-chat-with-expert + provider messages tools (lambda (_token) (void))))) + (record-current-usage! usage) + (make-assistant-message content tool-calls))) --- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -225,7 +225,8 @@ (def (provider-responder provider) (let ((backend (if (procedure? provider) provider - (make-provider-backend provider chat-with-expert)))) + (make-provider-backend + provider chat-with-expert-via-stream)))) (lambda (messages tool-specs step) (if (procedure? provider) (backend messages (verified-provider-tool-specs tool-specs) step) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -19,6 +19,7 @@ empty-text-tool-call-candidate uuid-text-tool-call-candidate openai-usage->alist + provider-retryable-error? ;; Grok Responses adapter (Phase 4) — exported so unit tests can drive ;; the pure helpers without making real HTTP calls. responses-parse-response @@ -108,6 +109,10 @@ (string-contains msg "524") (string-contains msg "529") (string-contains msg "connection closed before HTTP status") + ;; The non-streaming watchdog uses this wording. As with a stream + ;; timeout, no assistant response was accepted, so replaying the + ;; identical request is safe. + (string-contains msg "HTTP read timed out") (string-contains msg "stream read timed out") ;; Some OpenAI-compatible gateways occasionally return HTTP 200 with ;; a truncated or whitespace-only body. No assistant response was @@ -115,6 +120,10 @@ (and (string-contains msg "read-json") (string-contains msg "unexpected EOF"))))) +(def (provider-retryable-error? e) + ;; Public pure-policy hook used by deterministic provider regressions. + (and (retryable-error? e) #t)) + (def (api-call-with-retry thunk) (retry/predicate thunk retryable-error? 3 1.0)) @@ -2378,10 +2387,9 @@ (tool-call-arguments tc))) tool-calls))))) (values content tool-calls - (list (cons 'tokens-in (or (hash-get usage-acc "prompt_tokens") 0)) - (cons 'tokens-out (or (hash-get usage-acc "completion_tokens") 0)) - (cons 'cache-read (openai-cached-tokens usage-acc)) - (cons 'cost cost)) + ;; Keep streaming accounting aligned with non-streaming calls, + ;; including cache creation and reasoning token details. + (openai-usage->alist provider usage-acc) (build-stats (unbox finish-reason-box) (unbox logprob-box) (unbox entropy-box))))))) --- a/test/run.ss +++ b/test/run.ss @@ -78,6 +78,18 @@ (set! skip-count (+ skip-count 1)) (printf " SKIP: ~a (~a)~n" desc reason)) +(let ([timeout-error + (guard (e [#t e]) + (error 'http-post-json + "HTTP read timed out after 600s of silence (host: openrouter.ai)"))] + [ordinary-error + (guard (e [#t e]) + (error 'http-post-json "invalid request body"))]) + (check! "non-stream HTTP silence is retryable" + (provider-retryable-error? timeout-error) #t) + (check! "ordinary provider errors remain terminal" + (provider-retryable-error? ordinary-error) #f)) + (define (section name) (printf "~n~a~n" name)) (define (write-test-output-file path proc . _opts) @@ -8129,16 +8141,18 @@ (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) (tcp-close srv)))) -;; Verified runs are non-streaming at the workflow seam. They must still use -;; the expert-aware wrapper and report OpenRouter usage to the same callback -;; consumed by CLI status/benchmark accounting. +;; Verified runs collect the response at the workflow seam, but use streaming +;; transport so provider progress bytes prevent long generations from looking +;; like dead non-streaming requests. Usage still reaches CLI accounting. (let* ([chat-body (string-append - "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"accounted\"}," - "\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":120," + "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"accounted\"},\"finish_reason\":null}]}\n\n" + "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":120," "\"completion_tokens\":11,\"cost\":0.0125," "\"prompt_tokens_details\":{\"cached_tokens\":80,\"cache_write_tokens\":7}," - "\"completion_tokens_details\":{\"reasoning_tokens\":5}}}\n")] + "\"completion_tokens_details\":{\"reasoning_tokens\":5}}}\n\n" + "data: [DONE]\n\n")] [srv (tcp-listen "127.0.0.1" 0)] [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] [captured (vector #f)] @@ -8146,7 +8160,7 @@ (dynamic-wind (lambda () (void)) (lambda () - (serve-one-captured-json! srv captured 200 chat-body) + (serve-one-captured-sse! srv captured chat-body) (let* ([p (make-provider "openrouter" "unit-key" "unit-model" base-url)] [result (parameterize ([current-expert-disabled #t] @@ -8157,6 +8171,10 @@ (check! "verified responder returns provider text" (and (text-response? result) (text-response-content result)) "accounted") + (check-pred! "verified responder requests streaming transport" + (vector-ref captured 0) + (lambda (request) + (and request (str-contains? request "\"stream\":true")))) (check! "verified responder reports prompt tokens" (and seen-usage (cdr (assoc 'tokens-in seen-usage))) 120) (check! "verified responder reports completion tokens"