Make scoring configuration reproducible
ober
f45a8e7651a54cbcb389e277f94949f2c1614132
--- a/README.md +++ b/README.md @@ -35,9 +35,12 @@ Supported options are `--config FILE`, `--count N`, `--all`, `--from REV`, `--format table|json|jsonl|markdown`, `--metadata-only`, and `--heuristics-only`. -`--config FILE` reads guarded JSON. Supported keys are `path`, `count`, +`--config FILE` reads guarded JSON. Supported scan keys are `path`, `count`, `format`, `from`, `to`, `file`, `min_lines`, `metadata_only`, and -`heuristics_only`. Later CLI flags override earlier config values. +`heuristics_only`. Supported scoring keys are `human_threshold`, +`ai_threshold`, `weight_text`, `weight_code`, `weight_structure`, +`weight_similarity`, `weight_history`, and `weight_baseline`. Later CLI flags +override earlier config values. ## What It Reads @@ -52,6 +55,7 @@ write to the scanned worktree, refs, notes, hooks, or config. JSON output includes: - `detector_version` +- `config_hash` - repository path - commit and parent IDs - author/committer-facing metadata --- a/main-binary.ss +++ b/main-binary.ss @@ -6,6 +6,8 @@ (def tab (integer->char 9)) (defstruct options (command path count format commit from to file min-lines metadata-only? heuristics-only?)) +(defstruct scan-config + (hash human-threshold ai-threshold text-weight code-weight structure-weight similarity-weight history-weight baseline-weight)) (defstruct signal (name category score weight confidence reason evidence limitations)) (defstruct finding (commit parent author-name author-email time subject files additions deletions @@ -292,6 +294,50 @@ (if (> (vector-ref weights i) 0) (bitwise-ior acc (ash 1 i)) acc)))) (def (popcount32 n) (let loop ([x n] [acc 0]) (if (= x 0) acc (loop (ash x -1) (+ acc (bitwise-and x 1)))))) (def (hamming32 a b) (popcount32 (bitwise-xor a b))) +(def (hash-string s) + (str (stable-hash32 s))) + +(def (default-scan-config) + (make-scan-config "default" 0.20 0.50 0.10 0.35 0.15 0.15 0.15 0.00)) + +(def (config-fingerprint cfg) + (hash-string (string-join + (list (str (scan-config-human-threshold cfg)) + (str (scan-config-ai-threshold cfg)) + (str (scan-config-text-weight cfg)) + (str (scan-config-code-weight cfg)) + (str (scan-config-structure-weight cfg)) + (str (scan-config-similarity-weight cfg)) + (str (scan-config-history-weight cfg)) + (str (scan-config-baseline-weight cfg))) + ":"))) + +(def (finalize-config cfg) + (make-scan-config (config-fingerprint cfg) + (scan-config-human-threshold cfg) + (scan-config-ai-threshold cfg) + (scan-config-text-weight cfg) + (scan-config-code-weight cfg) + (scan-config-structure-weight cfg) + (scan-config-similarity-weight cfg) + (scan-config-history-weight cfg) + (scan-config-baseline-weight cfg))) + +(def current-config (finalize-config (default-scan-config))) + +(def (weight-for-category cfg category fallback) + (cond [(same-public-string? category "text") (scan-config-text-weight cfg)] + [(same-public-string? category "code") (scan-config-code-weight cfg)] + [(same-public-string? category "structure") (scan-config-structure-weight cfg)] + [(same-public-string? category "similarity") (scan-config-similarity-weight cfg)] + [(same-public-string? category "history") (scan-config-history-weight cfg)] + [(same-public-string? category "baseline") (scan-config-baseline-weight cfg)] + [else fallback])) + +(def (with-config-weight cfg s) + (make-signal (signal-name s) (signal-category s) (signal-score s) + (weight-for-category cfg (signal-category s) (signal-weight s)) + (signal-confidence s) (signal-reason s) (signal-evidence s) (signal-limitations s))) (def (best-sim hash hashes) (if (null? hashes) 0.0 (- 1.0 (/ (apply min (map (lambda (h) (hamming32 hash h)) hashes)) 32.0)))) (def (similarity-signal lines hashes) @@ -338,8 +384,8 @@ (def (verdict score metadata note) (cond [(not (string-empty? note)) "recorded-ai-authorship"] [(pair? metadata) "metadata-indicated-agent"] - [(>= score 0.50) "likely-ai-assisted"] - [(>= score 0.20) "mixed-uncertain"] + [(>= score (scan-config-ai-threshold current-config)) "likely-ai-assisted"] + [(>= score (scan-config-human-threshold current-config)) "mixed-uncertain"] [else "likely-human-style"])) (def (prior-additions repo revs author-id current file) @@ -387,7 +433,7 @@ (car sim-pair) (history-signal adds time (parent-time repo parent) (author-times repo revs author-email)) (baseline-signal adds (prior-additions repo revs author-email rev file))) '())] - [signals (if metadata-only? '() raw-signals)] + [signals (if metadata-only? '() (map (lambda (s) (with-config-weight current-config s)) raw-signals))] [score (if metadata-only? 0.0 (aggregate-score signals))] [v (if heuristics-only? (verdict score '() "") (verdict score metadata note))]) (list (make-finding id parent author-name author-email time subject paths adds dels (length lines) note attribution metadata signals score v @@ -436,7 +482,8 @@ (cons 'weight (signal-weight s)) (cons 'confidence (signal-confidence s)) (cons 'reason (signal-reason s)) (cons 'evidence (signal-evidence s)) (cons 'limitations (signal-limitations s)))) (def (finding-json repo f) - (list (cons 'detector_version detector-version) (cons 'repository repo) (cons 'commit (finding-commit f)) + (list (cons 'detector_version detector-version) (cons 'config_hash (scan-config-hash current-config)) + (cons 'repository repo) (cons 'commit (finding-commit f)) (cons 'parent (finding-parent f)) (cons 'author_name (finding-author-name f)) (cons 'author_email (finding-author-email f)) (cons 'time (finding-time f)) (cons 'subject (finding-subject f)) (cons 'files (finding-files f)) (cons 'additions (finding-additions f)) (cons 'deletions (finding-deletions f)) (cons 'added_lines (finding-added-lines f)) @@ -446,7 +493,8 @@ (cons 'metadata_hits (finding-metadata f)) (cons 'signals (map signal-json (finding-signals f))) (cons 'score (finding-score f)) (cons 'verdict (finding-verdict f)) (cons 'warnings (finding-warnings f)))) (def (report-json repo findings) - (list (cons 'detector_version detector-version) (cons 'repository repo) (cons 'count (length findings)) + (list (cons 'detector_version detector-version) (cons 'config_hash (scan-config-hash current-config)) + (cons 'repository repo) (cons 'count (length findings)) (cons 'findings (map (lambda (f) (finding-json repo f)) findings)))) (def (display-json repo findings) (displayln (json-string (report-json repo findings)))) @@ -505,21 +553,34 @@ (def (read-config-object path) (let ([content (try-result (read-file-string path))]) (if (ok? content) (parse-json-object (unwrap content)) #f))) +(def (config-scan-config obj) + (finalize-config + (make-scan-config "pending" + (config-number obj "human_threshold" (scan-config-human-threshold current-config)) + (config-number obj "ai_threshold" (scan-config-ai-threshold current-config)) + (config-number obj "weight_text" (scan-config-text-weight current-config)) + (config-number obj "weight_code" (scan-config-code-weight current-config)) + (config-number obj "weight_structure" (scan-config-structure-weight current-config)) + (config-number obj "weight_similarity" (scan-config-similarity-weight current-config)) + (config-number obj "weight_history" (scan-config-history-weight current-config)) + (config-number obj "weight_baseline" (scan-config-baseline-weight current-config))))) (def (apply-config opts path) (let ([obj (read-config-object path)]) (if (hash-table? obj) - (make-options (options-command opts) - (config-string obj "path" (options-path opts)) - (config-number obj "count" (options-count opts)) - (config-string obj "format" (options-format opts)) - (options-commit opts) - (config-string obj "from" (options-from opts)) - (config-string obj "to" (options-to opts)) - (config-string obj "file" (options-file opts)) - (config-number obj "min_lines" (options-min-lines opts)) - (config-bool obj "metadata_only" (options-metadata-only? opts)) - (config-bool obj "heuristics_only" (options-heuristics-only? opts))) + (begin + (set! current-config (config-scan-config obj)) + (make-options (options-command opts) + (config-string obj "path" (options-path opts)) + (config-number obj "count" (options-count opts)) + (config-string obj "format" (options-format opts)) + (options-commit opts) + (config-string obj "from" (options-from opts)) + (config-string obj "to" (options-to opts)) + (config-string obj "file" (options-file opts)) + (config-number obj "min_lines" (options-min-lines opts)) + (config-bool obj "metadata_only" (options-metadata-only? opts)) + (config-bool obj "heuristics_only" (options-heuristics-only? opts)))) opts))) (def (parse-options args) (let loop ([xs args] [opts (default-options)] [path-set? #f]) --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -36,6 +36,7 @@ git -C "$fixture" notes --ref=ai add -m '{"tool":"codex","model":"gpt-5","lines" json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 2) printf '%s\n' "$json" | grep -q '"count":2' +printf '%s\n' "$json" | grep -q '"config_hash":"[0-9]' printf '%s\n' "$json" | grep -q '"verdict":"recorded-ai-authorship"' printf '%s\n' "$json" | grep -q '"recorded_ai_note_present":true' printf '%s\n' "$json" | grep -q '"recorded_attribution":\[{"source":"refs/notes/ai","tool":"codex","model":"gpt-5","session":"","path":"src/generated.py","start":1,"end":360}\]' @@ -94,6 +95,11 @@ config_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file") printf '%s\n' "$config_json" | grep -q '"count":1' printf '%s\n' "$config_json" | grep -q '"files":\["src/generated.py"\]' +printf '{"format":"json","count":1,"weight_code":0.0,"weight_text":0.0,"weight_structure":0.0,"weight_similarity":0.0,"weight_history":0.0,"ai_threshold":0.9,"human_threshold":0.1}\n' > "$config_file" +weighted_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" --heuristics-only) +printf '%s\n' "$weighted_json" | grep -q '"config_hash":"[0-9]' +printf '%s\n' "$weighted_json" | grep -q '"category":"code","score":[0-9.]*,"weight":0.0' + printf 'not json\n' > "$config_file" bad_config_table=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" --count 1) printf '%s\n' "$bad_config_table" | grep -q 'commit score verdict'