provider: set sampling defaults for mlx to break LoRA repetition
ober
f54ad7f12ea30c7e03af3fbecea3051ce0dfcde2
--- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -472,6 +472,17 @@ ;;; OpenAI-compatible API ;;; +;; Local mlx-lm LoRAs (e.g. jerboa-lora) regress to a degenerate fixed +;; point at temperature=0 with no repetition_penalty: an 8x repetition of +;; the greeting then a slide into training-distribution prose. mlx_lm.server +;; accepts these fields on the OpenAI endpoint; cloud providers reject +;; non-standard ones, so gate this on provider name. +(def (apply-mlx-sampling! provider body) + (when (equal? (provider-name provider) "mlx") + (hash-put! body "temperature" 0.3) + (hash-put! body "top_p" 0.9) + (hash-put! body "repetition_penalty" 1.05))) + (def (openai-chat provider messages tools) (let* ((url (string-append (provider-base-url provider) "/chat/completions")) (headers (openai-headers provider)) @@ -499,6 +510,7 @@ (let ((body (make-hash-table))) (hash-put! body "model" (provider-model provider)) (hash-put! body "max_tokens" 32768) + (apply-mlx-sampling! provider body) (hash-put! body "messages" (map message->json messages)) (when (and tools (not (null? tools))) (hash-put! body "tools" tools) @@ -781,6 +793,7 @@ (hash-put! body "model" (provider-model provider)) (hash-put! body "stream" #t) (hash-put! body "max_tokens" 32768) + (apply-mlx-sampling! provider body) ;; Request usage data in stream (let ((opts (make-hash-table))) (hash-put! opts "include_usage" #t)