Add CLI threshold overrides
ober
8bcbd4b461b2e656dadca4ee1ae3937cb70612bc
--- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Supported options are `--config FILE`, `--count N`, `--all`, `--from REV`, `--to REV`, `--commit REV`, `--file PATH`, `--include PATH`, `--exclude PREFIX`, `--min-lines N`, `--max-files N`, `--max-added-lines N`, `--max-note-bytes N`, +`--threshold HUMAN,AI`, `--threshold HUMAN,UNCERTAIN,AI`, `--format table|json|jsonl|markdown`, `--metadata-only`, and `--heuristics-only`. @@ -45,6 +46,13 @@ Supported options are `--config FILE`, `--count N`, `--all`, `--from REV`, `max_added_lines`, and `max_note_bytes`. Later CLI flags override earlier config values. +Threshold CLI values override the active config for the current invocation. +The scanner has two operative cutoffs: scores below `HUMAN` are +`likely-human-style`, scores from `HUMAN` to below `AI` are `mixed-uncertain`, +and scores at or above `AI` are `likely-ai-assisted`. The three-value form is +accepted for compatibility with `human,uncertain,ai` wording; the middle value +is retained as a label boundary only and does not create a separate verdict. + By default, scans skip `vendor/`, `generated/`, `dist/`, `node_modules/`, and `.git/` paths. Use `--include PATH` or `--file PATH` to inspect one of those paths explicitly. --- a/main-binary.ss +++ b/main-binary.ss @@ -23,7 +23,7 @@ xs))) (def (usage) - (displayln "usage: jerboa main-binary.ss scan [PATH] [--config FILE] [--count N|--all] [--from REV] [--to REV] [--file PATH|--include PATH] [--exclude PREFIX] [--min-lines N] [--max-files N] [--max-added-lines N] [--max-note-bytes N] [--format table|json|jsonl|markdown]") + (displayln "usage: jerboa main-binary.ss scan [PATH] [--config FILE] [--count N|--all] [--from REV] [--to REV] [--file PATH|--include PATH] [--exclude PREFIX] [--min-lines N] [--threshold HUMAN,AI] [--max-files N] [--max-added-lines N] [--max-note-bytes N] [--format table|json|jsonl|markdown]") (displayln " jerboa main-binary.ss explain REV [PATH] [--format json|markdown|table]") (displayln " jerboa main-binary.ss stats [PATH] [--count N]") (displayln " jerboa main-binary.ss verify-authorship [PATH] [--count N]")) @@ -44,6 +44,31 @@ (for/fold ([n 0]) ([ch (in-string s)]) (+ (* n 10) (- (char->integer ch) (char->integer #\0)))) fallback)) +(def (decimal-score-string? s) + (let loop ([i 0] [saw-digit? #f] [saw-dot? #f]) + (if (= i (string-length s)) + saw-digit? + (let ([ch (string-ref s i)]) + (cond [(and (char>=? ch #\0) (char<=? ch #\9)) + (loop (+ i 1) #t saw-dot?)] + [(and (char=? ch #\.) (not saw-dot?)) + (loop (+ i 1) saw-digit? #t)] + [else #f]))))) + +(def (decimal-score s fallback) + (if (decimal-score-string? s) + (let loop ([i 0] [acc 0.0] [div 1.0] [fraction? #f]) + (if (= i (string-length s)) + acc + (let ([ch (string-ref s i)]) + (if (char=? ch #\.) + (loop (+ i 1) acc div #t) + (let ([digit (- (char->integer ch) (char->integer #\0))]) + (if fraction? + (let ([next-div (* div 10.0)]) + (loop (+ i 1) (+ acc (/ digit next-div)) next-div fraction?)) + (loop (+ i 1) (+ (* acc 10.0) digit) div fraction?))))))) + fallback)) (def (option-flag? s) (and (>= (string-length s) 2) @@ -698,6 +723,28 @@ (if (same-public-string? field "max-files") n (scan-config-max-files current-config)) (if (same-public-string? field "max-added-lines") n (scan-config-max-added-lines current-config)) (if (same-public-string? field "max-note-bytes") n (scan-config-max-note-bytes current-config))))))) +(def (update-thresholds value) + (let* ([parts (filter (lambda (part) (not (blank? part))) (string-split value #\,))] + [human (decimal-score (safe-ref parts 0 "") #f)] + [ai (decimal-score (if (>= (length parts) 3) (safe-ref parts 2 "") (safe-ref parts 1 "")) #f)]) + (if (and human ai (<= 0.0 human) (<= human ai) (<= ai 1.0)) + (begin + (set! current-config + (finalize-config + (make-scan-config "pending" + human + ai + (scan-config-text-weight current-config) + (scan-config-code-weight current-config) + (scan-config-structure-weight current-config) + (scan-config-similarity-weight current-config) + (scan-config-history-weight current-config) + (scan-config-baseline-weight current-config) + (scan-config-max-files current-config) + (scan-config-max-added-lines current-config) + (scan-config-max-note-bytes current-config)))) + #t) + #f))) (def (apply-config opts path) (let ([obj (read-config-object path)]) @@ -787,6 +834,10 @@ (begin (update-resource-limit "max-note-bytes" (cadr xs)) (loop (cddr xs) opts path-set?))] + [(and (string=? (car xs) "--threshold") (pair? (cdr xs))) + (begin + (update-thresholds (cadr xs)) + (loop (cddr xs) opts path-set?))] [(and (string=? (car xs) "--min-lines") (pair? (cdr xs))) (loop (cddr xs) (make-options (options-command opts) (options-path opts) (options-count opts) (options-format opts) --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -138,6 +138,12 @@ weighted_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" printf '%s\n' "$weighted_json" | grep -q '"config_hash":"[0-9]' printf '%s\n' "$weighted_json" | grep -q '"category":"code","score":[0-9.]*,"weight":0.0' +threshold_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --heuristics-only --threshold 0.99,1.0) +printf '%s\n' "$threshold_json" | grep -q '"verdict":"likely-human-style"' + +threshold_triplet_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --heuristics-only --threshold 0.99,0.995,1.0) +printf '%s\n' "$threshold_triplet_json" | grep -q '"verdict":"likely-human-style"' + printf 'not json\n' > "$config_file" bad_config_table=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" --count 1) printf '%s\n' "$bad_config_table" | grep -q 'commit score verdict'