Add strategy catalog parity audit
ober
763b489b1e224bce601a16392cbf02f5880fab08
--- a/GAPS.md +++ b/GAPS.md @@ -173,11 +173,15 @@ Cadence models strategies as named units with category, confidence, description, reason, and evidence. This project has signal structs but no registry/report inventory. +Status: implemented in stats JSON/text as `strategy_catalog`, including +enabled status, category, default weight, config field, limitations, and an +`uncataloged_strategies` audit. + Acceptance criteria: - Add a detector/strategy catalog in JSON stats or a new command. - Include enabled/disabled status, category, default weight, limitations, and - config key for each strategy. + config field for each strategy. - Test that every emitted signal has a catalog entry. ### G-021: Diff-shape heuristics are incomplete --- a/main-binary.ss +++ b/main-binary.ss @@ -1048,6 +1048,25 @@ (for ([w (finding-warnings f)]) (displayln (str "- " w)))))) (def stats-verdicts '("recorded-ai-authorship" "metadata-indicated-agent" "likely-ai-assisted" "mixed-uncertain" "likely-human-style" "insufficient-evidence")) (def stats-signal-categories '("text" "code" "structure" "semantic" "similarity" "history" "baseline")) +(def (strategy-catalog-entry name category default-weight config-field limitations) + (list (cons 'name name) + (cons 'category category) + (cons 'enabled #t) + (cons 'default_weight default-weight) + (cons 'config_field config-field) + (cons 'limitations limitations))) +(def strategy-catalog + (list (strategy-catalog-entry "message-style" "text" 0.10 "weight_text" "message style alone is weak evidence") + (strategy-catalog-entry "lexical-code-style" "code" 0.35 "weight_code" "language-neutral lexical analysis is noisy") + (strategy-catalog-entry "cadence-code-patterns" "code" 0.18 "weight_code" "framework conventions and mechanical scaffolds can match") + (strategy-catalog-entry "diff-structure" "structure" 0.15 "weight_structure" "formatters, generated files, and refactors can look similar") + (strategy-catalog-entry "cadence-diff-shape" "structure" 0.15 "weight_structure" "large generated files, vendored updates, and mechanical refactors can match") + (strategy-catalog-entry "semantic-alignment" "semantic" 0.10 "weight_semantic" "offline lexical overlap is a weak proxy for semantic self-summary") + (strategy-catalog-entry "simhash-similarity" "similarity" 0.15 "weight_similarity" "similarity in one scan window cannot prove original authorship") + (strategy-catalog-entry "history-timing" "history" 0.15 "weight_history" "Git commit timestamps are not typing timestamps") + (strategy-catalog-entry "author-baseline" "baseline" 0.00 "weight_baseline" "missing or anomalous baseline is contextual evidence") + (strategy-catalog-entry "repository-baseline" "baseline" 0.00 "weight_baseline" "repository baseline uses commit-size history only") + (strategy-catalog-entry "local-llm-opinion" "model" 0.10 "provider" "local model output is secondary evidence and may be wrong"))) (def (alist-ref/default pairs name fallback) (let ([hit (assoc name pairs)]) @@ -1087,6 +1106,13 @@ (append-map finding-signals findings)) (def (signal-names findings) (unique (map signal-name (all-signals findings)))) +(def (catalog-entry-name entry) + (alist-ref/default entry 'name "")) +(def (cataloged-strategy? name) + (for/or ([entry strategy-catalog]) + (same-public-string? (catalog-entry-name entry) name))) +(def (uncataloged-strategies findings) + (filter (lambda (name) (not (cataloged-strategy? name))) (signal-names findings))) (def (signals-with-name findings name) (filter (lambda (s) (same-public-string? (signal-name s) name)) (all-signals findings))) (def (strategy-hit-json findings name) @@ -1119,6 +1145,8 @@ (cons 'likely_ai_assisted_by_heuristics (count-verdict findings "likely-ai-assisted")) (cons 'score_bands (map (lambda (v) (score-band-json findings v)) stats-verdicts)) (cons 'signal_coverage (map (lambda (category) (signal-coverage-json findings category)) stats-signal-categories)) + (cons 'strategy_catalog strategy-catalog) + (cons 'uncataloged_strategies (uncataloged-strategies findings)) (cons 'strategy_hits (map (lambda (name) (strategy-hit-json findings name)) (signal-names findings))) (cons 'warning_counts (map (lambda (warning) (warning-count-json findings warning)) (unique (all-warnings findings))))))) @@ -1139,6 +1167,11 @@ (displayln "signal coverage:") (for ([category stats-signal-categories]) (displayln (str " " category ": " (count-where (lambda (f) (finding-has-signal-category? f category)) findings)))) + (displayln "strategy catalog:") + (displayln (str " entries: " (length strategy-catalog))) + (displayln (str " uncataloged: " (if (null? (uncataloged-strategies findings)) + "none" + (string-join (uncataloged-strategies findings) ",")))) (displayln "strategy hits:") (for ([name (signal-names findings)]) (let ([signals (signals-with-name findings name)]) --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -166,6 +166,8 @@ printf '%s\n' "$stats" | grep -q 'signal coverage:' printf '%s\n' "$stats" | grep -q ' code: 2' printf '%s\n' "$stats" | grep -q ' semantic: 2' printf '%s\n' "$stats" | grep -q 'strategy hits:' +printf '%s\n' "$stats" | grep -q 'strategy catalog:' +printf '%s\n' "$stats" | grep -q ' uncataloged: none' printf '%s\n' "$stats" | grep -q ' message-style:' printf '%s\n' "$stats" | grep -q ' cadence-diff-shape:' printf '%s\n' "$stats" | grep -q 'warnings:' @@ -180,6 +182,9 @@ printf '%s\n' "$stats_json" | grep -q '"signal_coverage":\[{"category":"text","c printf '%s\n' "$stats_json" | grep -q '{"category":"semantic","count":2}' printf '%s\n' "$stats_json" | grep -q '"strategy_hits":\[{"name":"message-style","count":2' printf '%s\n' "$stats_json" | grep -q '{"name":"cadence-diff-shape","count":1' +printf '%s\n' "$stats_json" | grep -q '"strategy_catalog":\[{"name":"message-style","category":"text","enabled":true' +printf '%s\n' "$stats_json" | grep -q '{"name":"local-llm-opinion","category":"model","enabled":true' +printf '%s\n' "$stats_json" | grep -q '"uncataloged_strategies":\[\]' printf '%s\n' "$stats_json" | grep -q '"warning_counts":\[' jsonl=$("$root/bin/jerboa-aigit" scan "$fixture" --format jsonl --count 2)