Add strategy hit stats parity
ober
c9206934e5d211e25ee8311e64226d4c19b4ef30
--- a/GAPS.md +++ b/GAPS.md @@ -231,6 +231,9 @@ Acceptance criteria: Cadence reports strategy coverage and hit counts. This project reports signal category coverage, but not strategy hit rates or category confidence summaries. +Status: implemented for strategy hit counts, average score by strategy, +category coverage, and warning counts in both text and JSON stats output. + Acceptance criteria: - Add stats for strategy hit count, average score by strategy, category --- a/main-binary.ss +++ b/main-binary.ss @@ -1052,6 +1052,23 @@ (def (signal-coverage-json findings category) (list (cons 'category category) (cons 'count (count-where (lambda (f) (finding-has-signal-category? f category)) findings)))) +(def (all-signals findings) + (append-map finding-signals findings)) +(def (signal-names findings) + (unique (map signal-name (all-signals findings)))) +(def (signals-with-name findings name) + (filter (lambda (s) (same-public-string? (signal-name s) name)) (all-signals findings))) +(def (strategy-hit-json findings name) + (let* ([signals (signals-with-name findings name)] + [positive (filter (lambda (s) (> (signal-score s) 0.0)) signals)]) + (list (cons 'name name) + (cons 'count (length positive)) + (cons 'average_score (if (null? signals) 0.0 (mean (map signal-score signals))))))) +(def (all-warnings findings) + (append-map finding-warnings findings)) +(def (warning-count-json findings warning) + (list (cons 'warning warning) + (cons 'count (count-where (lambda (w) (same-public-string? w warning)) (all-warnings findings))))) (def (stats-json repo findings) (let ([tools (stats-tools findings)]) @@ -1070,7 +1087,9 @@ (cons 'metadata_indicated_agents (count-where (lambda (f) (pair? (finding-metadata f))) findings)) (cons 'likely_ai_assisted_by_heuristics (count-verdict findings "likely-ai-assisted")) (cons 'score_bands (map (lambda (v) (score-band-json findings v)) stats-verdicts)) - (cons 'signal_coverage (map (lambda (category) (signal-coverage-json findings category)) stats-signal-categories))))) + (cons 'signal_coverage (map (lambda (category) (signal-coverage-json findings category)) stats-signal-categories)) + (cons 'strategy_hits (map (lambda (name) (strategy-hit-json findings name)) (signal-names findings))) + (cons 'warning_counts (map (lambda (warning) (warning-count-json findings warning)) (unique (all-warnings findings))))))) (def (display-stats-json repo findings) (displayln (json-string (stats-json repo findings)))) @@ -1088,7 +1107,20 @@ (displayln (str " " v ": " (count-where (lambda (f) (same-public-string? (finding-verdict f) v)) findings)))) (displayln "signal coverage:") (for ([category stats-signal-categories]) - (displayln (str " " category ": " (count-where (lambda (f) (finding-has-signal-category? f category)) findings)))))) + (displayln (str " " category ": " (count-where (lambda (f) (finding-has-signal-category? f category)) findings)))) + (displayln "strategy hits:") + (for ([name (signal-names findings)]) + (let ([signals (signals-with-name findings name)]) + (displayln (str " " name ": " + (length (filter (lambda (s) (> (signal-score s) 0.0)) signals)) + " avg=" (if (null? signals) 0.0 (mean (map signal-score signals))))))) + (displayln "warnings:") + (let ([warnings (unique (all-warnings findings))]) + (if (null? warnings) + (displayln " none") + (for ([warning warnings]) + (displayln (str " " warning ": " + (count-where (lambda (w) (same-public-string? w warning)) (all-warnings findings))))))))) (def (default-options) (make-options "scan" "." 50 "table" #f #f #f #f 0 #f #f)) (def (config-value obj field fallback) --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -163,6 +163,10 @@ printf '%s\n' "$stats" | grep -q ' recorded-ai-authorship: 1' printf '%s\n' "$stats" | grep -q 'signal coverage:' printf '%s\n' "$stats" | grep -q ' code: 2' printf '%s\n' "$stats" | grep -q ' semantic: 2' +printf '%s\n' "$stats" | grep -q 'strategy hits:' +printf '%s\n' "$stats" | grep -q ' message-style:' +printf '%s\n' "$stats" | grep -q ' cadence-diff-shape:' +printf '%s\n' "$stats" | grep -q 'warnings:' stats_json=$("$root/bin/jerboa-aigit" stats "$fixture" --count 2 --format json) printf '%s\n' "$stats_json" | grep -q '"count":2' @@ -172,6 +176,9 @@ printf '%s\n' "$stats_json" | grep -q '"recorded_ai_lines":360' printf '%s\n' "$stats_json" | grep -q '"score_bands":\[{"name":"recorded-ai-authorship","count":1}' printf '%s\n' "$stats_json" | grep -q '"signal_coverage":\[{"category":"text","count":2}' printf '%s\n' "$stats_json" | grep -q '{"category":"semantic","count":2}' +printf '%s\n' "$stats_json" | grep -q '"strategy_hits":\[{"name":"message-style","count":2' +printf '%s\n' "$stats_json" | grep -q '{"name":"cadence-diff-shape","count":1' +printf '%s\n' "$stats_json" | grep -q '"warning_counts":\[' jsonl=$("$root/bin/jerboa-aigit" scan "$fixture" --format jsonl --count 2) jsonl_lines=$(printf '%s\n' "$jsonl" | wc -l | tr -d ' ')