Add safe local LLM provider adapter
ober
3a410162d13a9b7f8a18889ea7c64e2f28cac4d9
--- a/README.md +++ b/README.md @@ -54,10 +54,17 @@ and scores at or above `AI` are `likely-ai-assisted`. The three-value form is accepted for compatibility with `human,uncertain,ai` wording; the middle value is retained as a label boundary only and does not create a separate verdict. -`--no-llm` is the default. `--provider NAME` records the requested provider in -JSON output. `--llm` is accepted as an explicit request, but this release has no -active provider adapter; it emits a warning and still reports `llm_used:false` -and `network_used:false`. +`--no-llm` is the default. `--llm --provider local` enables an optional local +provider adapter when either `local_provider_command` is set in the JSON config +as an argv array or `JERBOA_AIGIT_LOCAL_PROVIDER` points at an executable. The +scanner sends one bounded JSON payload as the final argv item and expects strict +JSON on stdout with a numeric `score` plus optional `reason` and `evidence`. +The provider executable must be an absolute path outside the scanned repository; +repo-local provider commands are refused so scanning never executes repository +code. +Local provider output is reported as a secondary `model` signal; it never +overwrites recorded authorship metadata. Remote/network providers are not +implemented in this release, and `network_used` remains `false`. `stats` reports commit count, distinct authors, recorded tools, recorded AI line counts, metadata/heuristic counts, score-band counts, and signal-category --- a/main-binary.ss +++ b/main-binary.ss @@ -511,6 +511,8 @@ (def current-llm-used? #f) (def current-network-used? #f) (def current-llm-requested? #f) +(def current-local-provider-command '()) +(def local-provider-input-line-limit 80) (def (set-provider! name) (set! current-requested-provider (if (string-empty? name) "none" name))) @@ -524,10 +526,101 @@ (set! current-network-used? #f) (set! current-analysis-provider "offline-heuristics")) -(def (llm-warnings) - (if current-llm-requested? - (list (str "LLM analysis requested for provider " current-requested-provider "; no provider adapter is active, using offline heuristics")) +(def (llm-warnings) '()) +(def (set-local-provider-command! parts) + (if (list? parts) + (set! current-local-provider-command (filter string? parts)))) +(def (local-provider-command) + (if (pair? current-local-provider-command) + current-local-provider-command + (let ([path (getenv "JERBOA_AIGIT_LOCAL_PROVIDER")]) + (if (and (string? path) (not (string-empty? path))) + (list path) + '())))) +(def (local-provider-payload repo rev subject paths lines signals score) + (json-string + (list (cons 'detector_version detector-version) + (cons 'repository repo) + (cons 'commit rev) + (cons 'subject subject) + (cons 'files paths) + (cons 'heuristic_score score) + (cons 'signals (map signal-json signals)) + (cons 'added_lines_sample (bounded-list lines local-provider-input-line-limit))))) +(def (local-provider-input-warnings lines) + (if (> (length lines) local-provider-input-line-limit) + (list (str "local provider input added-line sample truncated to " local-provider-input-line-limit " lines")) '())) +(def (provider-evidence obj) + (let ([items (hash-get/default obj "evidence" '())] + [reason (hash-get/default obj "reason" "")]) + (let ([strings (if (list? items) (filter string? items) '())]) + (cond [(pair? strings) strings] + [(and (string? reason) (not (string-empty? reason))) (list reason)] + [else '()])))) +(def (provider-score obj) + (let ([score (hash-get/default obj "score" #f)]) + (if (number? score) (clamp01 score) #f))) +(def (provider-reason obj) + (let ([reason (hash-get/default obj "reason" "")]) + (if (string? reason) reason "local provider returned secondary AI-authorship score"))) +(def (local-provider-signal obj) + (let ([score (provider-score obj)]) + (if score + (list (sig "local-llm-opinion" "model" score 0.10 "low" + (provider-reason obj) + (provider-evidence obj) + "local model output is secondary evidence and may be wrong")) + '()))) +(def (macos-var-alias? path) + (and (> (string-length path) 5) + (char=? (string-ref path 0) #\/) + (char=? (string-ref path 1) #\v) + (char=? (string-ref path 2) #\a) + (char=? (string-ref path 3) #\r) + (char=? (string-ref path 4) #\/))) +(def (macos-private-var-path path) + (if (macos-var-alias? path) + (str "/private" path) + path)) +(def (list-prefix-same? prefix xs) + (cond [(null? prefix) #t] + [(null? xs) #f] + [(same-public-string? (car prefix) (car xs)) + (list-prefix-same? (cdr prefix) (cdr xs))] + [else #f])) +(def (path-inside-root? path root) + (list-prefix-same? (string-split (macos-private-var-path root) #\/) + (string-split (macos-private-var-path path) #\/))) +(def (local-provider-command-warning repo cmd) + (cond [(null? cmd) + "local provider requested but no local_provider_command config or JERBOA_AIGIT_LOCAL_PROVIDER is set"] + [(not (path-absolute? (car cmd))) + "local provider command must be an absolute path outside the scanned repository"] + [(path-inside-root? (car cmd) repo) + "local provider command inside the scanned repository was refused"] + [else #f])) +(def (local-provider-result repo rev subject paths lines signals score) + (cond [(not current-llm-requested?) (list '() '())] + [(not (same-public-string? current-requested-provider "local")) + (list '() (list (str "LLM analysis requested for provider " current-requested-provider "; no adapter is active, using offline heuristics")))] + [else + (let* ([cmd (local-provider-command)] + [command-warning (local-provider-command-warning repo cmd)]) + (if command-warning + (list '() (list command-warning)) + (let* ([payload (local-provider-payload repo rev subject paths lines signals score)] + [output (try-result (run-process (append cmd (list payload))))]) + (if (ok? output) + (let ([obj (parse-json-object (unwrap output))]) + (if (and (hash-table? obj) (provider-score obj)) + (begin + (set! current-analysis-provider "offline-heuristics+local-llm") + (set! current-llm-used? #t) + (set! current-network-used? #f) + (list (local-provider-signal obj) (local-provider-input-warnings lines))) + (list '() '("local provider returned malformed JSON or omitted numeric score")))) + (list '() '("local provider command failed; using offline heuristics"))))))])) (def (weight-for-category cfg category fallback) (cond [(same-public-string? category "text") (scan-config-text-weight cfg)] @@ -705,14 +798,18 @@ (baseline-signal adds (prior-additions repo revs author-email rev file)) (repository-baseline-signal adds (prior-repository-additions repo revs rev file))) '())] - [signals (if metadata-only? '() (map (lambda (s) (with-config-weight current-config s)) raw-signals))] + [weighted-signals (if metadata-only? '() (map (lambda (s) (with-config-weight current-config s)) raw-signals))] + [preliminary-score (if metadata-only? 0.0 (aggregate-score weighted-signals))] + [provider-pair (local-provider-result repo id subject paths lines weighted-signals preliminary-score)] + [signals (append weighted-signals (car provider-pair))] [score (if metadata-only? 0.0 (aggregate-score signals))] [insufficient? (and (null? signals) (null? metadata) (string-empty? note))] [v (cond [insufficient? "insufficient-evidence"] [heuristics-only? (verdict score '() "")] [else (verdict score metadata note)])]) (list (make-finding id parent author-name author-email time subject paths adds dels (length lines) note attribution metadata signals score v - (warnings files lines note note-obj metadata-only? heuristics-only? min-lines file-count line-count note-count parents shallow? binary-count rename-copy-count missing-parent?)) + (append (warnings files lines note note-obj metadata-only? heuristics-only? min-lines file-count line-count note-count parents shallow? binary-count rename-copy-count missing-parent?) + (cadr provider-pair))) (cadr sim-pair)))) (def (scan-repo repo revs file min-lines metadata-only? heuristics-only?) @@ -1012,6 +1109,7 @@ (begin (set! current-config (config-scan-config obj)) (set! current-excludes (append (config-string-list obj "exclude" '()) current-excludes)) + (set-local-provider-command! (config-string-list obj "local_provider_command" current-local-provider-command)) (make-options (options-command opts) (config-string obj "path" (options-path opts)) (config-number obj "count" (options-count opts)) --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -4,7 +4,8 @@ set -eu root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) fixture=$(mktemp -d) shallow=$(mktemp -d) -trap 'rm -rf "$fixture" "$shallow"' EXIT +provider_tmp=$(mktemp -d) +trap 'rm -rf "$fixture" "$shallow" "$provider_tmp"' EXIT git -C "$fixture" init -q git -C "$fixture" config user.name "Human Dev" @@ -76,10 +77,41 @@ llm_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --ll printf '%s\n' "$llm_json" | grep -q '"provider":"local"' printf '%s\n' "$llm_json" | grep -q '"llm_used":false' printf '%s\n' "$llm_json" | grep -q '"network_used":false' -printf '%s\n' "$llm_json" | grep -q 'LLM analysis requested for provider local; no provider adapter is active, using offline heuristics' +printf '%s\n' "$llm_json" | grep -q 'local provider requested but no local_provider_command config or JERBOA_AIGIT_LOCAL_PROVIDER is set' + +repo_provider_script="$fixture/local-provider.sh" +{ + printf '#!/usr/bin/env sh\n' + printf 'printf '\''{"score":0.91,"reason":"repo provider should be refused"}\\n'\''\n' +} > "$repo_provider_script" +chmod +x "$repo_provider_script" +repo_provider_config="$fixture/repo-local-provider.json" +printf '{"local_provider_command":["%s"]}\n' "$repo_provider_script" > "$repo_provider_config" +repo_provider_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$repo_provider_config" --format json --count 1 --llm --provider local) +printf '%s\n' "$repo_provider_json" | grep -q '"llm_used":false' +printf '%s\n' "$repo_provider_json" | grep -q 'local provider command inside the scanned repository was refused' + +provider_script="$provider_tmp/local-provider.sh" +{ + printf '#!/usr/bin/env sh\n' + printf 'case "$1" in\n' + printf ' *generated.py*) printf '\''{"score":0.91,"reason":"fixture local provider","evidence":["provider saw generated.py"]}\\n'\'' ;;\n' + printf ' *) printf '\''{"score":0.10,"reason":"fixture local provider"}\\n'\'' ;;\n' + printf 'esac\n' +} > "$provider_script" +chmod +x "$provider_script" +provider_config="$fixture/local-provider.json" +printf '{"local_provider_command":["%s"]}\n' "$provider_script" > "$provider_config" +local_llm_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$provider_config" --format json --count 1 --llm --provider local) +printf '%s\n' "$local_llm_json" | grep -q '"analysis_provider":"offline-heuristics+local-llm"' +printf '%s\n' "$local_llm_json" | grep -q '"llm_used":true' +printf '%s\n' "$local_llm_json" | grep -q '"network_used":false' +printf '%s\n' "$local_llm_json" | grep -q '"name":"local-llm-opinion"' +printf '%s\n' "$local_llm_json" | grep -q '"category":"model"' +printf '%s\n' "$local_llm_json" | grep -q '"evidence":\["provider saw generated.py"\]' no_llm_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --llm --provider local --no-llm) -if printf '%s\n' "$no_llm_json" | grep -q 'LLM analysis requested'; then +if printf '%s\n' "$no_llm_json" | grep -q 'local provider requested'; then echo "no-llm should disable requested LLM warning" >&2 exit 1 fi