forge phase 3: per-model sampling map + policy + error hierarchy
ober
aac2bffe2a2ab243a37aec48e67c7d30d2f29e46
--- a/build-binary.ss +++ b/build-binary.ss @@ -110,6 +110,7 @@ '("lib/jcode/core/models" "lib/jcode/core/config" "lib/jcode/core/log" + "lib/jcode/core/errors" "lib/jcode/core/session" "lib/jcode/core/message" "lib/jcode/core/secrets" @@ -136,6 +137,7 @@ "lib/jcode/guardrails/validator" "lib/jcode/guardrails/respond" "lib/jcode/guardrails/guardrails" + "lib/jcode/provider/sampling" "lib/jcode/provider/provider" "lib/jcode/tool/registry" "lib/jcode/tool/file" new file mode 100644 --- /dev/null +++ b/src/jcode/core/errors.ss @@ -0,0 +1,46 @@ +;;; jcode forge error hierarchy +;;; +;;; Faithful port of forge's exception hierarchy (forge/errors.py). The root +;;; is &forge-error (forge's ForgeError); each forge-specific error derives +;;; from it, so a caller can catch the whole family with forge-error? or one +;;; member with its own predicate. Every raiser compounds a message-condition +;;; (so the error prints through display-condition like any other) and a +;;; who-condition naming the origin. +;;; +;;; This module grows per phase: it currently carries the errors the ported +;;; surfaces actually raise. ToolResolutionError is deliberately NOT a +;;; forge-error in forge (it is the privileged "retry with different args" +;;; signal); it will be added as a plain condition when the tool loop needs it. +;;; +;;; guard gotcha: clauses MUST use [#t ...], never `else` — guard captures the +;;; `else` keyword. Use (guard (e [(unsupported-model-error? e) ...] [#t ...])). + +(export &forge-error make-forge-error forge-error? + &unsupported-model unsupported-model-error? + unsupported-model-error-model + raise-unsupported-model) + +;; Root of the forge family. Derives from R6RS &error so error? holds and it +;; threads through the existing message-condition?/condition-message handling. +(define-condition-type &forge-error &error + make-forge-error forge-error?) + +;; UnsupportedModelError — raised by apply-sampling-defaults in strict mode for +;; a model with no row in the sampling map. Failing loud is intentional: strict +;; declares "I want this model's card profile"; silently falling through to +;; backend defaults would defeat that intent. +(define-condition-type &unsupported-model &forge-error + make-unsupported-model unsupported-model-error? + (model unsupported-model-error-model)) + +(def (raise-unsupported-model model) + (raise + (condition + (make-unsupported-model model) + (make-who-condition 'apply-sampling-defaults) + (make-message-condition + (string-append + "No recommended sampling defaults registered for model " + model + ". Either add an entry to the sampling map (with HF card URL) " + "or drop strict sampling."))))) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -25,6 +25,7 @@ :jcode/core/log :jcode/core/message :jcode/core/models + :jcode/provider/sampling :jerboa/core :jerboa/runtime) @@ -563,16 +564,11 @@ ;;; OpenAI-compatible API ;;; -;; Local mlx-lm LoRAs (e.g. jerboa-lora) regress to a degenerate fixed -;; point at temperature=0 with no repetition_penalty: an 8x repetition of -;; the greeting then a slide into training-distribution prose. mlx_lm.server -;; accepts these fields on the OpenAI endpoint; cloud providers reject -;; non-standard ones, so gate this on provider name. -(def (apply-mlx-sampling! provider body) - (when (equal? (provider-name provider) "mlx") - (hash-put! body "temperature" 0.3) - (hash-put! body "top_p" 0.9) - (hash-put! body "repetition_penalty" 1.05))) +;; Per-model recommended sampling now lives in :jcode/provider/sampling +;; (apply-sampling-to-body!): it splats the model's HF-card profile onto the +;; body under the session policy (/forge sampling on|off|strict, default on), +;; and falls back to the legacy mlx-lm LoRA safety net when no card row +;; applies. The fixed-point-at-temperature=0 LoRA guard is preserved there. ;; --- logprobs / confidence stats --- ;; Opt-in via config: expert.escalation.request_logprobs = true. Cheap to @@ -773,7 +769,7 @@ (let ((body (make-hash-table))) (hash-put! body "model" (provider-model provider)) (hash-put! body "max_tokens" 32768) - (apply-mlx-sampling! provider body) + (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (apply-logprobs! body) (hash-put! body "messages" (map message->json messages)) (when (and tools (not (null? tools)) @@ -1131,7 +1127,7 @@ (hash-put! body "model" (provider-model provider)) (hash-put! body "stream" #t) (hash-put! body "max_tokens" 32768) - (apply-mlx-sampling! provider body) + (apply-sampling-to-body! body (provider-model provider) (provider-name provider)) (apply-logprobs! body) ;; Request usage data in stream (let ((opts (make-hash-table))) new file mode 100644 --- /dev/null +++ b/src/jcode/provider/sampling.ss @@ -0,0 +1,201 @@ +;;; jcode per-model sampling defaults +;;; +;;; Faithful port of forge/clients/sampling_defaults.py. Per-model recommended +;;; sampling params, sourced one model at a time from the live HF model card +;;; (the inline URL on each row is the provenance — do not add rows without +;;; fetching the card). Two pure-ish layers, mirroring forge: +;;; +;;; get-sampling-defaults(model) pure lookup -> fresh copy or {}. +;;; apply-sampling-defaults(model strict) policy: the forge 4-quadrant table +;;; +;;; | strict | in map | behavior | +;;; |--------|--------|-----------------------------------| +;;; | #t | yes | return the row (fresh copy) | +;;; | #t | no | raise &unsupported-model | +;;; | #f | yes | one-shot INFO log; return {} | +;;; | #f | no | return {} (silent) | +;;; +;;; Because jcode's guardrails wrap *every* provider, the session policy +;;; (forge-sampling-policy) defaults to 'on — apply the card values when the +;;; model is known, fall through to backend defaults when it is not, never +;;; break an unknown model. 'strict opts into forge's recommended_sampling=True +;;; (raise on unknown); 'off applies nothing. This supersedes the old +;;; apply-mlx-sampling! hardcode (which survives only as the local-LoRA safety +;;; net for mlx models that have no card row). + +(export get-sampling-defaults + apply-sampling-defaults + model-sampling-known? + forge-sampling-policy + apply-sampling-to-body!) + +(import :jcode/core/errors + :jcode/core/log) + +(def logger (make-logger "sampling")) + +;; Build a sampling-param row from alternating key/value args. +(def (srow . kvs) + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) + h + (begin (hash-put! h (car xs) (cadr xs)) + (loop (cddr xs))))))) + +;; Each model is keyed once per identity form the caller might use (Ollama- +;; style string, GGUF stem, llamafile stem). All forms are independent rows so +;; vendor-specific guidance can diverge from the HF card without forcing +;; alignment. Values are verified one model at a time against the live card. +(def model-sampling-defaults + (let ((m (make-hash-table))) + ;; Qwen3 — thinking-mode values (forge runs these in thinking mode). + (hash-put! m "qwen3:4b-instruct-2507-q4_K_M" (srow "temperature" 0.7 "top_p" 0.8 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507 + (hash-put! m "qwen3:4b-thinking-2507-q4_K_M" (srow "temperature" 0.6 "top_p" 0.95 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507 + (hash-put! m "qwen3:8b-q4_K_M" (srow "temperature" 0.6 "top_p" 0.95 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-8B + (hash-put! m "Qwen3-8B-Q4_K_M" (srow "temperature" 0.6 "top_p" 0.95 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-8B + (hash-put! m "qwen3:8b-q8_0" (srow "temperature" 0.6 "top_p" 0.95 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-8B + (hash-put! m "Qwen3-8B-Q8_0" (srow "temperature" 0.6 "top_p" 0.95 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-8B + (hash-put! m "qwen3:14b-q4_K_M" (srow "temperature" 0.6 "top_p" 0.95 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-14B + (hash-put! m "Qwen3-14B-Q4_K_M" (srow "temperature" 0.6 "top_p" 0.95 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-14B + ;; Qwen3.5/3.6 — thinking-mode general-tasks profile. For precise-coding swap + ;; temperature=0.6 and presence_penalty=0.0 (other keys unchanged). + (hash-put! m "qwen3.5:27b-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 20 "min_p" 0.0 "presence_penalty" 1.5)) ; https://huggingface.co/Qwen/Qwen3.5-27B + (hash-put! m "Qwen3.5-27B-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 20 "min_p" 0.0 "presence_penalty" 1.5)) ; https://huggingface.co/Qwen/Qwen3.5-27B + (hash-put! m "qwen3.5:35b-a3b-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 20 "min_p" 0.0 "presence_penalty" 1.5)) ; https://huggingface.co/Qwen/Qwen3.5-35B-A3B + (hash-put! m "Qwen3.5-35B-A3B-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 20 "min_p" 0.0 "presence_penalty" 1.5)) ; https://huggingface.co/Qwen/Qwen3.5-35B-A3B + (hash-put! m "qwen3.6:35b-a3b-ud-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 20 "min_p" 0.0 "presence_penalty" 1.5)) ; https://huggingface.co/Qwen/Qwen3.6-35B-A3B + (hash-put! m "Qwen3.6-35B-A3B-UD-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 20 "min_p" 0.0 "presence_penalty" 1.5)) ; https://huggingface.co/Qwen/Qwen3.6-35B-A3B + ;; Qwen3-Coder — non-thinking instruct; card omits min_p / presence_penalty. + (hash-put! m "qwen3-coder:30b-a3b-instruct-q4_K_M" (srow "temperature" 0.7 "top_p" 0.8 "top_k" 20 "repeat_penalty" 1.05)) ; https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct + (hash-put! m "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M" (srow "temperature" 0.7 "top_p" 0.8 "top_k" 20 "repeat_penalty" 1.05)) ; https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct + ;; Qwen3-Next 80B-A3B-Instruct — hybrid attention MoE; thinking-mode profile. + (hash-put! m "qwen3-next:80b-a3b-instruct-q4_K_M" (srow "temperature" 0.7 "top_p" 0.8 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct + (hash-put! m "Qwen3-Next-80B-A3B-Instruct-Q4_K_M" (srow "temperature" 0.7 "top_p" 0.8 "top_k" 20 "min_p" 0.0)) ; https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct + ;; Qwen3-Coder-Next — coder fine-tune over Qwen3-Next-80B-A3B base; differs + ;; notably from Qwen3-Coder-30B (no min_p / repeat_penalty / presence). + (hash-put! m "qwen3-coder-next:80b-a3b-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 40)) ; https://huggingface.co/Qwen/Qwen3-Coder-Next + (hash-put! m "Qwen3-Coder-Next-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 40)) ; https://huggingface.co/Qwen/Qwen3-Coder-Next + ;; Gemma 4 — one standardized profile for all use cases. + (hash-put! m "gemma4:31b-it-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-31b-it + (hash-put! m "gemma-4-31B-it-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-31b-it + (hash-put! m "gemma4:26b-a4b-it-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-26b-a4b-it + (hash-put! m "gemma-4-26B-A4B-it-UD-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-26b-a4b-it + (hash-put! m "gemma4:26b-a4b-it-q8_0" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-26b-a4b-it + (hash-put! m "gemma-4-26B-A4B-it-Q8_0" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-26b-a4b-it + (hash-put! m "gemma4:e4b-it-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-e4b-it + (hash-put! m "gemma-4-E4B-it-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-e4b-it + (hash-put! m "gemma4:e4b-it-q8_0" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-e4b-it + (hash-put! m "gemma-4-E4B-it-Q8_0" (srow "temperature" 1.0 "top_p" 0.95 "top_k" 64)) ; https://huggingface.co/google/gemma-4-e4b-it + ;; Mistral Small 4 — high-effort profile (T=0.7 + reasoning_effort="high"). + (hash-put! m "mistral-small-4:119b-2603-q4_K_M" (srow "temperature" 0.7 "chat_template_kwargs" (srow "reasoning_effort" "high"))) ; https://huggingface.co/mistralai/Mistral-Small-4-119B-2603 + (hash-put! m "Mistral-Small-4-119B-2603-UD-Q4_K_M" (srow "temperature" 0.7 "chat_template_kwargs" (srow "reasoning_effort" "high"))) ; https://huggingface.co/mistralai/Mistral-Small-4-119B-2603 + ;; Qwen3.5-122B-A10B — instruct-mode "balanced" preset (reasoning-budget 0). + (hash-put! m "Qwen3.5-122B-A10B-Q4_K_M" (srow "temperature" 0.7 "top_p" 0.8 "top_k" 20)) ; https://huggingface.co/Qwen/Qwen3.5-122B-A10B + ;; gpt-oss-120b — reasoning_effort medium; do NOT set repeat/presence penalty. + (hash-put! m "gpt-oss:120b-q4_K_M" (srow "temperature" 1.0 "top_p" 1.0 "top_k" 0 "min_p" 0.0 "chat_template_kwargs" (srow "reasoning_effort" "medium"))) ; https://huggingface.co/openai/gpt-oss-120b + https://github.com/ggml-org/llama.cpp/discussions/15396 + (hash-put! m "gpt-oss-120b-Q4_K_M" (srow "temperature" 1.0 "top_p" 1.0 "top_k" 0 "min_p" 0.0 "chat_template_kwargs" (srow "reasoning_effort" "medium"))) ; https://huggingface.co/openai/gpt-oss-120b + https://github.com/ggml-org/llama.cpp/discussions/15396 + ;; NVIDIA Nemotron-3-Super-120B-A12B — enable_thinking + low_effort + force_nonempty_content. + (hash-put! m "nemotron-3-super:120b-a12b-q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "chat_template_kwargs" (srow "enable_thinking" #t "low_effort" #t "force_nonempty_content" #t))) ; https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + (hash-put! m "NVIDIA-Nemotron-3-Super-120B-A12B-UD-Q4_K_M" (srow "temperature" 1.0 "top_p" 0.95 "chat_template_kwargs" (srow "enable_thinking" #t "low_effort" #t "force_nonempty_content" #t))) ; https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + ;; NVIDIA Nemotron-3-Nano-30B-A3B — deterministic tool-calling preset. + (hash-put! m "Nemotron-3-Nano-30B-A3B-Q4_K_M" (srow "temperature" 0.6 "top_p" 0.95 "chat_template_kwargs" (srow "enable_thinking" #t))) ; https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + ;; Mistral Small 3.2 & Devstral Small 2 — cards only specify temperature. + (hash-put! m "mistral-small-3.2:24b-instruct-2506-q4_K_M" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506 + (hash-put! m "Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506 + (hash-put! m "mistral-small-3.2:24b-instruct-2506-q8_0" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506 + (hash-put! m "Mistral-Small-3.2-24B-Instruct-2506-Q8_0" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506 + (hash-put! m "devstral-small-2:24b-instruct-2512-q4_K_M" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512 + (hash-put! m "Devstral-Small-2-24B-Instruct-2512-Q4_K_M" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512 + (hash-put! m "devstral-small-2:24b-instruct-2512-q8_0" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512 + (hash-put! m "Devstral-Small-2-24B-Instruct-2512-Q8_0" (srow "temperature" 0.15)) ; https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512 + ;; Ministral-3 Instruct — card says temperature below 0.1; 0.05 picked. + (hash-put! m "ministral-3:8b-instruct-2512-q4_K_M" (srow "temperature" 0.05)) ; https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512 (card: temp<0.1) + (hash-put! m "Ministral-3-8B-Instruct-2512-Q4_K_M" (srow "temperature" 0.05)) ; https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512 (card: temp<0.1) + (hash-put! m "ministral-3:8b-instruct-2512-q8_0" (srow "temperature" 0.05)) ; https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512 (card: temp<0.1) + (hash-put! m "Ministral-3-8B-Instruct-2512-Q8_0" (srow "temperature" 0.05)) ; https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512 (card: temp<0.1) + (hash-put! m "ministral-3:14b-instruct-2512-q4_K_M" (srow "temperature" 0.05)) ; https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512 (card: temp<0.1) + (hash-put! m "Ministral-3-14B-Instruct-2512-Q4_K_M" (srow "temperature" 0.05)) ; https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512 (card: temp<0.1) + ;; Ministral-3 Reasoning — specific temperature per size (top_p omitted). + (hash-put! m "ministral-3:8b-reasoning-2512-q4_K_M" (srow "temperature" 0.7)) ; https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512 + (hash-put! m "Ministral-3-8B-Reasoning-2512-Q4_K_M" (srow "temperature" 0.7)) ; https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512 + (hash-put! m "ministral-3:8b-reasoning-2512-q8_0" (srow "temperature" 0.7)) ; https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512 + (hash-put! m "Ministral-3-8B-Reasoning-2512-Q8_0" (srow "temperature" 0.7)) ; https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512 + (hash-put! m "ministral-3:14b-reasoning-2512-q4_K_M" (srow "temperature" 1.0)) ; https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512 + (hash-put! m "Ministral-3-14B-Reasoning-2512-Q4_K_M" (srow "temperature" 1.0)) ; https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512 + ;; Mistral Nemo — formal recommendation temp=0.3. + (hash-put! m "mistral-nemo:12b-instruct-2407-q4_K_M" (srow "temperature" 0.3)) ; https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407 + (hash-put! m "Mistral-Nemo-Instruct-2407-Q4_K_M" (srow "temperature" 0.3)) ; https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407 (GGUF) + (hash-put! m "Mistral-Nemo-Instruct-2407.Q4_K_M" (srow "temperature" 0.3)) ; https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407 (llamafile) + ;; Granite 4.0 — IBM-pointed reference cites greedy decoding (T=0). + (hash-put! m "granite-4.0:h-micro-q4_K_M" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; https://unsloth.ai/docs/models/tutorials/ibm-granite-4.0 (cites IBM) + (hash-put! m "granite-4.0-h-micro-Q4_K_M" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; https://unsloth.ai/docs/models/tutorials/ibm-granite-4.0 (cites IBM) + (hash-put! m "granite-4.0:h-tiny-q4_K_M" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; https://unsloth.ai/docs/models/tutorials/ibm-granite-4.0 (cites IBM) + (hash-put! m "granite-4.0-h-tiny-Q4_K_M" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; https://unsloth.ai/docs/models/tutorials/ibm-granite-4.0 (cites IBM) + ;; Granite 4.1 — UNCONFIRMED; mirrors granite-4.0 IBM convention (greedy). + (hash-put! m "granite4.1:8b-q4_K_M" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; unconfirmed; mirrors granite-4.0 IBM convention + (hash-put! m "granite-4.1-8b-Q4_K_M" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; unconfirmed; mirrors granite-4.0 IBM convention + (hash-put! m "granite4.1:8b-q8_0" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; unconfirmed; mirrors granite-4.0 IBM convention + (hash-put! m "granite-4.1-8b-Q8_0" (srow "temperature" 0.0 "top_p" 1.0 "top_k" 0)) ; unconfirmed; mirrors granite-4.0 IBM convention + ;; Intentionally absent (no formal recommendation from any official source): + ;; llama3.1:* mistral:7b-instruct-v0.3 phi-4 (base). These fall through + ;; to the unknown-model path (backend defaults). + m)) + +;; Models for which the one-shot non-strict INFO log already fired this process. +(def sampling-info-logged (make-hash-table)) + +(def (shallow-copy-hash src) + (let ((h (make-hash-table))) + (hash-for-each (lambda (k v) (hash-put! h k v)) src) + h)) + +;; Pure lookup — no logging, no raising. Fresh shallow copy or {} for unknowns. +(def (get-sampling-defaults model) + (let ((row (hash-ref model-sampling-defaults model #f))) + (if row (shallow-copy-hash row) (make-hash-table)))) + +(def (model-sampling-known? model) + (hash-key? model-sampling-defaults model)) + +;; forge's apply_sampling_defaults — the 4-quadrant policy layer. +(def (apply-sampling-defaults model strict?) + (let ((in-map (hash-key? model-sampling-defaults model))) + (cond + (strict? + (if in-map + (get-sampling-defaults model) + (raise-unsupported-model model))) + (else + (when (and in-map (not (hash-key? sampling-info-logged model))) + (log-info logger "recommended-sampling-available" + `((model . ,model) + (hint . "set /forge sampling strict to use this model's card profile"))) + (hash-put! sampling-info-logged model #t)) + (make-hash-table))))) + +;; Session policy for jcode's always-on layer: +;; 'off apply nothing +;; 'on apply the card row if known, else fall through (never raise) — default +;; 'strict forge recommended_sampling=True (apply if known, raise if unknown) +(def forge-sampling-policy (make-parameter 'on)) + +(def (sampling-params-for model) + (let ((policy (forge-sampling-policy))) + (cond + ((eq? policy 'off) (make-hash-table)) + ((eq? policy 'strict) (apply-sampling-defaults model #t)) + (else (get-sampling-defaults model))))) + +;; Splat the resolved sampling params onto an OpenAI-style request body. When +;; no row applies (unknown model, or policy 'off), preserve the legacy mlx +;; safety net: local mlx-lm LoRAs regress to a degenerate fixed point at +;; temperature=0 with no repetition penalty, so nudge them off it. +(def (apply-sampling-to-body! body model provider-name) + (let ((params (sampling-params-for model))) + (if (pair? (hash-keys params)) + (hash-for-each (lambda (k v) (hash-put! body k v)) params) + (when (equal? provider-name "mlx") + (hash-put! body "temperature" 0.3) + (hash-put! body "top_p" 0.9) + (hash-put! body "repetition_penalty" 1.05))))) --- a/src/jcode/ui/cli.ss +++ b/src/jcode/ui/cli.ss @@ -25,6 +25,7 @@ :jcode/tool/web :jcode/tool/batch :jcode/tool/git + :jcode/provider/sampling :jcode/mcp/client :jcode/tool/lsp :jcode/core/plugin @@ -280,13 +281,19 @@ EXAMPLES: (if (forge-respond-enforced?) "ON (bare text retried as a tool call)" "OFF (bare text is a normal final answer)")) - (printf "Toggle with /forge on | /forge off~n")) + (printf " sampling ~a~n" + (let ((p (forge-sampling-policy))) + (cond + ((eq? p 'off) "off (no per-model sampling)") + ((eq? p 'strict) "strict (use card profile; error on unknown model)") + (else "on (use card profile if known, else backend defaults)")))) + (printf "Toggle with /forge on | /forge off | /forge sampling off|on|strict~n")) (def (handle-command input session-id) (let ((cmd (string-trim (substring input 1 (string-length input))))) (cond ((equal? cmd "help") - (display "\nCommands:\n /help Show this help\n /model [name] Show or set model\n /provider [name] Show or set provider\n /plan Switch to PLAN mode (read-only)\n /build Switch to BUILD mode (read+write)\n /mode Show current mode\n /mcp Toggle MCP tools on/off\n /tools List available tools\n /clear Start a new session\n /sessions List saved sessions\n /compact Show message count\n /undo [N] Revert last N checkpoint(s) (default 1)\n /checkpoints List recent shadow-git checkpoints\n /forge [on|off] Show or toggle forge guardrails\n /quit Exit\n\nMulti-line: end a line with \\ to continue on the next line.\n\n")) + (display "\nCommands:\n /help Show this help\n /model [name] Show or set model\n /provider [name] Show or set provider\n /plan Switch to PLAN mode (read-only)\n /build Switch to BUILD mode (read+write)\n /mode Show current mode\n /mcp Toggle MCP tools on/off\n /tools List available tools\n /clear Start a new session\n /sessions List saved sessions\n /compact Show message count\n /undo [N] Revert last N checkpoint(s) (default 1)\n /checkpoints List recent shadow-git checkpoints\n /forge [on|off] Show or toggle forge guardrails\n /forge sampling <off|on|strict> Per-model sampling policy\n /quit Exit\n\nMulti-line: end a line with \\ to continue on the next line.\n\n")) ((equal? cmd "model") (printf "Provider: ~a~n" (or (current-provider-override) (config-provider))) (printf "Model: ~a~n" (or (current-model-override) (config-model))) @@ -380,6 +387,15 @@ EXAMPLES: ((or (equal? cmd "forge off") (equal? cmd "forge enforce off")) (forge-respond-enforced? #f) (printf "Forge respond-forcing: OFF (bare text is a normal final answer).~n")) + ((equal? cmd "forge sampling off") + (forge-sampling-policy 'off) + (printf "Forge sampling: OFF (no per-model sampling params applied).~n")) + ((or (equal? cmd "forge sampling on") (equal? cmd "forge sampling")) + (forge-sampling-policy 'on) + (printf "Forge sampling: ON (apply the model's HF-card profile if known, else backend defaults).~n")) + ((equal? cmd "forge sampling strict") + (forge-sampling-policy 'strict) + (printf "Forge sampling: STRICT (apply card profile; error on a model with no card row).~n")) (#t ;; Slash dispatch: builtins win over file-based skills. (let* ((space-pos (string-index cmd #\space)) --- a/test/run.ss +++ b/test/run.ss @@ -14,7 +14,9 @@ (jcode guardrails rescue) (jcode guardrails validator) (jcode guardrails respond) - (jcode guardrails guardrails)) + (jcode guardrails guardrails) + (jcode core errors) + (jcode provider sampling)) ;; ── Helpers ────────────────────────────────────────────────────── @@ -535,6 +537,105 @@ (let ([g (make-guardrails '("ls" "respond") 3 2 #t '("respond"))]) (check! "facade record terminal reached" (guardrails-record g '("respond")) #t)) +;; ── sampling defaults ────────────────────────────────────────────── +(section "=== sampling defaults ===") + +;; jcode hashes are raw Chez equal? hashtables (the get-/apply- functions +;; return them, and bodies are built the same way the `args` helper does), so +;; inspect them with the Chez hashtable API. + +;; pure lookup: known model returns its card row +(let ([row (get-sampling-defaults "qwen3:8b-q4_K_M")]) + (check! "sampling known: temperature" (hashtable-ref row "temperature" #f) 0.6) + (check! "sampling known: top_p" (hashtable-ref row "top_p" #f) 0.95) + (check! "sampling known: top_k int" (hashtable-ref row "top_k" #f) 20) + (check! "sampling known: min_p" (hashtable-ref row "min_p" #f) 0.0)) + +;; unknown model: empty map, never raises +(check! "sampling unknown: empty" + (hashtable-size (get-sampling-defaults "gpt-4o")) 0) + +;; known? predicate +(check! "sampling known? yes" (model-sampling-known? "Qwen3-8B-Q4_K_M") #t) +(check! "sampling known? no" (model-sampling-known? "no-such-model") #f) + +;; lookup returns a fresh copy — mutating it must not corrupt the map +(let ([r1 (get-sampling-defaults "qwen3:8b-q4_K_M")]) + (hashtable-set! r1 "temperature" 99.0) + (let ([r2 (get-sampling-defaults "qwen3:8b-q4_K_M")]) + (check! "sampling fresh copy each call" + (and (not (eq? r1 r2)) (hashtable-ref r2 "temperature" #f)) 0.6))) + +;; nested chat_template_kwargs preserved (gpt-oss reasoning_effort) +(let* ([row (get-sampling-defaults "gpt-oss:120b-q4_K_M")] + [ctk (hashtable-ref row "chat_template_kwargs" #f)]) + (check! "sampling nested top_k zero" (hashtable-ref row "top_k" #f) 0) + (check-pred! "sampling nested ctk is hash" ctk hashtable?) + (check! "sampling nested reasoning_effort" + (hashtable-ref ctk "reasoning_effort" #f) "medium")) + +;; boolean values survive in nested kwargs (nemotron enable_thinking) +(let* ([row (get-sampling-defaults "Nemotron-3-Nano-30B-A3B-Q4_K_M")] + [ctk (hashtable-ref row "chat_template_kwargs" #f)]) + (check! "sampling nested boolean #t" (hashtable-ref ctk "enable_thinking" #f) #t)) + +;; 4-quadrant policy +(check! "sampling policy strict+known: temperature" + (hashtable-ref (apply-sampling-defaults "qwen3:8b-q4_K_M" #t) "temperature" #f) 0.6) +(check! "sampling policy nonstrict+known: empty" + (hashtable-size (apply-sampling-defaults "qwen3:8b-q4_K_M" #f)) 0) +(check! "sampling policy nonstrict+unknown: empty" + (hashtable-size (apply-sampling-defaults "no-such-model" #f)) 0) +(check! "sampling policy strict+unknown: raises" + (guard (e [(unsupported-model-error? e) 'raised] [#t 'other]) + (apply-sampling-defaults "no-such-model" #t)) + 'raised) +(check! "sampling policy strict+unknown: error carries model" + (guard (e [(unsupported-model-error? e) (unsupported-model-error-model e)] [#t #f]) + (apply-sampling-defaults "no-such-model" #t)) + "no-such-model") +;; unsupported-model is a member of the forge-error family +(check! "sampling unsupported-model is a forge-error" + (guard (e [(forge-error? e) #t] [#t #f]) + (apply-sampling-defaults "no-such-model" #t)) + #t) + +;; body application under the session policy +;; 'on + known: splat the card profile onto the body +(let ([body (make-hashtable equal-hash equal?)]) + (parameterize ([forge-sampling-policy 'on]) + (apply-sampling-to-body! body "qwen3:8b-q4_K_M" "mlx")) + (check! "body on+known: temperature" (hashtable-ref body "temperature" #f) 0.6) + (check! "body on+known: top_k" (hashtable-ref body "top_k" #f) 20)) + +;; 'on + unknown mlx model: legacy LoRA safety net (0.3 / 0.9 / 1.05) +(let ([body (make-hashtable equal-hash equal?)]) + (parameterize ([forge-sampling-policy 'on]) + (apply-sampling-to-body! body "jerboa-mlx-lora" "mlx")) + (check! "body on+unknown mlx: temp net" (hashtable-ref body "temperature" #f) 0.3) + (check! "body on+unknown mlx: rep net" + (hashtable-ref body "repetition_penalty" #f) 1.05)) + +;; 'on + unknown cloud model: nothing applied (no mlx net) +(let ([body (make-hashtable equal-hash equal?)]) + (parameterize ([forge-sampling-policy 'on]) + (apply-sampling-to-body! body "gpt-4o" "openai")) + (check! "body on+unknown cloud: untouched" (hashtable-size body) 0)) + +;; 'off: forge card sampling not applied even for a known model (cloud +;; provider, so the mlx safety net cannot confound the assertion) +(let ([body (make-hashtable equal-hash equal?)]) + (parameterize ([forge-sampling-policy 'off]) + (apply-sampling-to-body! body "qwen3:8b-q4_K_M" "openai")) + (check! "body off+known: untouched" (hashtable-size body) 0)) + +;; the mlx LoRA safety net is policy-independent: it still fires under 'off +;; for an mlx provider with no card row (degenerate-output guard) +(let ([body (make-hashtable equal-hash equal?)]) + (parameterize ([forge-sampling-policy 'off]) + (apply-sampling-to-body! body "jerboa-mlx-lora" "mlx")) + (check! "body off+mlx: net still fires" (hashtable-ref body "temperature" #f) 0.3)) + ;; ── Results ─────────────────────────────────────────────────────── (printf "~n~a passed, ~a failed~n" pass-count fail-count)