Raise local verified first draft token floor
ober
ea8f91cd8086f892dbe31d61556b11ba77b0ef48
--- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -6201,6 +6201,9 @@ (current-max-tokens-cap (and local-model? (local-verified-first-draft-cap))) + (current-max-tokens-floor + (and local-model? + (local-verified-first-draft-cap))) (current-verify-failure-count 0) (current-force-expert-next? #f) (current-serving-forced-expert? #f) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -13,6 +13,7 @@ provider-base-url provider-local? current-max-tokens-cap + current-max-tokens-floor current-tool-choice-override openai-function-tool-choice current-stream-abort? @@ -992,6 +993,12 @@ ;; cloud-origin workflows leave it unbound and are unaffected. (def current-max-tokens-cap (make-parameter #f)) +;; Verified local first drafts may need to raise a conservative provider +;; default or config entry enough to fit one complete structured write. This is +;; applied before the cap, so repair turns and cloud expert calls can still be +;; bounded tightly. +(def current-max-tokens-floor (make-parameter #f)) + ;; Verified workflows can temporarily require a specific OpenAI-compatible ;; function call after the model has inspected enough context. Leave unbound ;; for ordinary chat and providers that should choose tools normally. @@ -1017,8 +1024,12 @@ (if (local-provider? (provider-name provider)) *openai-local-default-max-tokens* *openai-default-max-tokens*))) + (floor (positive-int-value (current-max-tokens-floor))) + (floored (if (and floor (local-provider? (provider-name provider))) + (max configured floor) + configured)) (cap (positive-int-value (current-max-tokens-cap)))) - (if cap (min configured cap) configured))) + (if cap (min floored cap) floored))) (def (decimal-at s start) (let ((n (string-length s))) --- a/test/run.ss +++ b/test/run.ss @@ -10396,6 +10396,33 @@ (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) (tcp-close srv)))) +(let* ([chat-body "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n"] + [srv (tcp-listen "127.0.0.1" 0)] + [base-url (format "http://127.0.0.1:~a/v1" (tcp-server-port srv))] + [captured (vector #f)] + [cfg (make-hashtable equal-hash equal?)] + [providers (make-hashtable equal-hash equal?)] + [mlx (make-hashtable equal-hash equal?)] + [old-max-tokens (getenv "JCODE_MAX_TOKENS")]) + (hashtable-set! mlx "max_tokens" 1234) + (hashtable-set! providers "mlx" mlx) + (hashtable-set! cfg "providers" providers) + (dynamic-wind + (lambda () (putenv "JCODE_MAX_TOKENS" "")) + (lambda () + (serve-one-captured-json! srv captured 200 chat-body) + (let* ([p (make-provider "mlx" "" "unit-test-model" base-url)] + [_ (parameterize ([*config* cfg] + [current-max-tokens-floor 8192] + [current-max-tokens-cap 8192]) + (provider-chat p (list (make-user-message "hi")) '()))] + [req (vector-ref captured 0)]) + (check! "mlx local verified floor raises configured max_tokens" + (and req (str-contains? req "\"max_tokens\":8192")) #t))) + (lambda () + (putenv "JCODE_MAX_TOKENS" (or old-max-tokens "")) + (tcp-close srv)))) + ;; Verified runs collect the response at the workflow seam, but use streaming ;; transport so provider progress bytes prevent long generations from looking ;; like dead non-streaming requests. Usage still reaches CLI accounting.