Add threshold sensitivity evaluation
ober
1051762973949159a91444fb4e41eb092dcd8182
--- a/GAPS.md +++ b/GAPS.md @@ -84,6 +84,12 @@ Acceptance criteria: - Document why the default thresholds were selected. - Keep config hash/version stable and visible in reports. +Status: implemented with `eval/sensitivity.py`, candidate configs in +`eval/configs/`, and committed `eval/sensitivity-report.md`. The default is +retained because it ties candidate overall F1 on the sample while preserving +recorded-provenance precision and avoiding new false positives; evaluation +reports now include the scanner config hash per case. + ### G-004: No differential comparison against reference tools Parity with `sniff`, Cadence, and git-ai is not measured. --- a/eval/README.md +++ b/eval/README.md @@ -57,3 +57,13 @@ The report scans labeled cases and emits precision, recall, F1, false-positive rate, false-negative rate, and a confusion matrix by verdict band. It also breaks metrics out for recorded provenance, metadata clues, and heuristic-only findings. + +Run sensitivity comparisons: + +```sh +eval/sensitivity.py --output eval/sensitivity-report.md +``` + +This compares built-in defaults against `eval/configs/permissive.json` and +`eval/configs/strict.json`, preserving each finding's config hash in the +underlying evaluation report. new file mode 100644 --- /dev/null +++ b/eval/configs/permissive.json @@ -0,0 +1,10 @@ +{ + "human_threshold": 0.20, + "ai_threshold": 0.55, + "weight_text": 1.10, + "weight_code": 1.10, + "weight_structure": 1.00, + "weight_similarity": 1.00, + "weight_history": 1.00, + "weight_baseline": 1.00 +} new file mode 100644 --- /dev/null +++ b/eval/configs/strict.json @@ -0,0 +1,10 @@ +{ + "human_threshold": 0.40, + "ai_threshold": 0.78, + "weight_text": 0.90, + "weight_code": 0.95, + "weight_structure": 0.95, + "weight_similarity": 1.00, + "weight_history": 1.00, + "weight_baseline": 1.00 +} --- a/eval/report.py +++ b/eval/report.py @@ -27,9 +27,12 @@ def ensure_fixture(root, rows): subprocess.check_call([str(root / "eval/generate-fixtures.sh")], cwd=root) -def scan_case(root, scanner, row): +def scan_case(root, scanner, row, config): repo = root / row["repo"] if not Path(row["repo"]).is_absolute() else Path(row["repo"]) - out = run([str(scanner), "scan", str(repo), "--commit", row["rev"], "--format", "json"], cwd=root) + cmd = [str(scanner), "scan", str(repo), "--commit", row["rev"], "--format", "json"] + if config: + cmd.extend(["--config", str(config)]) + out = run(cmd, cwd=root) parsed = json.loads(out) findings = parsed.get("findings", []) if findings: @@ -114,6 +117,7 @@ def markdown_report(result): "# Evaluation report", "", f"Scanner: `{result['scanner']}`", + f"Config: `{result['config']}`", f"Labels: `{result['labels']}`", f"Cases: {len(result['cases'])}", "", @@ -132,11 +136,11 @@ def markdown_report(result): for key, count in sorted(result["confusion_by_verdict_band"].items()): truth, predicted = key.split("\t", 1) lines.append(f"| {truth} | {predicted} | {count} |") - lines.extend(["", "## Cases", "", "| case | label | mode | ambiguous | predicted band | verdict | score |", "|---|---|---|---|---|---|---:|"]) + lines.extend(["", "## Cases", "", "| case | label | mode | ambiguous | predicted band | verdict | score | config hash |", "|---|---|---|---|---|---|---:|---|"]) for case in result["cases"]: lines.append( f"| {case['case_id']} | {case['label']} | {case['mode']} | {case['ambiguous']} | " - f"{case['predicted_band']} | {case['verdict']} | {case['score']} |" + f"{case['predicted_band']} | {case['verdict']} | {case['score']} | {case['config_hash']} |" ) return "\n".join(lines) + "\n" @@ -145,6 +149,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--labels", default="eval/labels.tsv") parser.add_argument("--scanner", default="bin/jerboa-aigit") + parser.add_argument("--config") parser.add_argument("--format", choices=["markdown", "json"], default="markdown") parser.add_argument("--output") args = parser.parse_args() @@ -160,7 +165,8 @@ def main(): confusion = {} cases = [] for row in rows: - finding = scan_case(root, scanner, row) + config = root / args.config if args.config else None + finding = scan_case(root, scanner, row, config) band = verdict_band(finding) for name in slices: add_case(counts[name], truth_positive(row, name), predicted_positive(finding, name)) @@ -174,10 +180,12 @@ def main(): "predicted_band": band, "verdict": finding.get("verdict", ""), "score": finding.get("score", 0), + "config_hash": finding.get("config_hash", ""), }) result = { "scanner": args.scanner, + "config": args.config or "default", "labels": args.labels, "metrics": {name: rates(counts[name]) for name in slices}, "confusion_by_verdict_band": confusion, --- a/eval/sample-report.md +++ b/eval/sample-report.md @@ -1,6 +1,7 @@ # Evaluation report Scanner: `bin/jerboa-aigit` +Config: `default` Labels: `eval/labels.tsv` Cases: 10 @@ -30,15 +31,15 @@ Cases: 10 ## Cases -| case | label | mode | ambiguous | predicted band | verdict | score | -|---|---|---|---|---|---|---:| -| human_basic | human | non_ai | false | likely-human-style | likely-human-style | 0.045000000000000005 | -| ai_recorded_note | ai_assisted | recorded | false | recorded | recorded-ai-authorship | 1.0 | -| ai_heuristic_style | ai_assisted | heuristic | false | likely-ai-assisted | likely-ai-assisted | 1.0 | -| metadata_attested_author | metadata_attested | metadata | false | metadata | metadata-indicated-agent | 0.0 | -| bot_dependency_update | bot | non_ai | false | insufficient-evidence | insufficient-evidence | 0.0 | -| generated_vendor | generated_vendor | non_ai | false | insufficient-evidence | insufficient-evidence | 0.0 | -| formatter_only | formatter_only | non_ai | false | likely-human-style | likely-human-style | 0.0 | -| mechanical_refactor | refactor | non_ai | false | likely-human-style | likely-human-style | 0.1225 | -| merge_commit | merge | non_ai | false | likely-human-style | likely-human-style | 0.045000000000000005 | -| ambiguous_assistance | ambiguous | mixed | true | likely-human-style | likely-human-style | 0.0 | +| case | label | mode | ambiguous | predicted band | verdict | score | config hash | +|---|---|---|---|---|---|---:|---| +| human_basic | human | non_ai | false | likely-human-style | likely-human-style | 0.045000000000000005 | 7842737080095761057 | +| ai_recorded_note | ai_assisted | recorded | false | recorded | recorded-ai-authorship | 1.0 | 7842737080095761057 | +| ai_heuristic_style | ai_assisted | heuristic | false | likely-ai-assisted | likely-ai-assisted | 1.0 | 7842737080095761057 | +| metadata_attested_author | metadata_attested | metadata | false | metadata | metadata-indicated-agent | 0.0 | 7842737080095761057 | +| bot_dependency_update | bot | non_ai | false | insufficient-evidence | insufficient-evidence | 0.0 | 7842737080095761057 | +| generated_vendor | generated_vendor | non_ai | false | insufficient-evidence | insufficient-evidence | 0.0 | 7842737080095761057 | +| formatter_only | formatter_only | non_ai | false | likely-human-style | likely-human-style | 0.0 | 7842737080095761057 | +| mechanical_refactor | refactor | non_ai | false | likely-human-style | likely-human-style | 0.1225 | 7842737080095761057 | +| merge_commit | merge | non_ai | false | likely-human-style | likely-human-style | 0.045000000000000005 | 7842737080095761057 | +| ambiguous_assistance | ambiguous | mixed | true | likely-human-style | likely-human-style | 0.0 | 7842737080095761057 | new file mode 100644 --- /dev/null +++ b/eval/sensitivity-report.md @@ -0,0 +1,22 @@ +# Threshold and weight sensitivity + +This report compares the default detector configuration with two committed candidate configurations. +The fixture corpus is intentionally small; use this as a repeatable guardrail, not a final calibration study. + +## Summary + +| candidate | overall F1 | recorded F1 | metadata F1 | heuristic F1 | overall FP | overall FN | rationale | +|---|---:|---:|---:|---:|---:|---:|---| +| default | 1.000 | 1.000 | 0.667 | 1.000 | 0 | 0 | Current shipped thresholds and weights. | +| permissive | 1.000 | 1.000 | 0.667 | 1.000 | 0 | 0 | Lower AI threshold and slightly higher text/code weights. | +| strict | 1.000 | 1.000 | 0.667 | 1.000 | 0 | 0 | Higher AI threshold and slightly lower text/code/structure weights. | + +## Default selection + +The default remains selected when it ties the best overall F1 while keeping recorded-provenance precision at 1.0 and avoiding extra false positives on non-AI controls. On this sample, candidate changes do not improve the result enough to justify changing the visible config hash/threshold behavior. + +## Candidate configs + +- `default`: built-in defaults — Current shipped thresholds and weights. +- `permissive`: eval/configs/permissive.json — Lower AI threshold and slightly higher text/code weights. +- `strict`: eval/configs/strict.json — Higher AI threshold and slightly lower text/code/structure weights. new file mode 100755 --- /dev/null +++ b/eval/sensitivity.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +CANDIDATES = [ + ("default", None, "Current shipped thresholds and weights."), + ("permissive", "eval/configs/permissive.json", "Lower AI threshold and slightly higher text/code weights."), + ("strict", "eval/configs/strict.json", "Higher AI threshold and slightly lower text/code/structure weights."), +] + + +def run_json(root, config): + cmd = [str(root / "eval/report.py"), "--format", "json"] + if config: + cmd.extend(["--config", config]) + return json.loads(subprocess.check_output(cmd, cwd=root, text=True)) + + +def markdown(results): + lines = [ + "# Threshold and weight sensitivity", + "", + "This report compares the default detector configuration with two committed candidate configurations.", + "The fixture corpus is intentionally small; use this as a repeatable guardrail, not a final calibration study.", + "", + "## Summary", + "", + "| candidate | overall F1 | recorded F1 | metadata F1 | heuristic F1 | overall FP | overall FN | rationale |", + "|---|---:|---:|---:|---:|---:|---:|---|", + ] + for name, config, rationale, result in results: + metrics = result["metrics"] + overall = metrics["overall"] + lines.append( + f"| {name} | {overall['f1']:.3f} | {metrics['recorded']['f1']:.3f} | " + f"{metrics['metadata']['f1']:.3f} | {metrics['heuristic']['f1']:.3f} | " + f"{overall['fp']} | {overall['fn']} | {rationale} |" + ) + lines.extend([ + "", + "## Default selection", + "", + "The default remains selected when it ties the best overall F1 while keeping recorded-provenance precision at 1.0 and avoiding extra false positives on non-AI controls. On this sample, candidate changes do not improve the result enough to justify changing the visible config hash/threshold behavior.", + "", + "## Candidate configs", + "", + ]) + for name, config, rationale, _result in results: + lines.append(f"- `{name}`: {config or 'built-in defaults'} — {rationale}") + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="eval/sensitivity-report.md") + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + results = [] + for name, config, rationale in CANDIDATES: + results.append((name, config, rationale, run_json(root, config))) + text = markdown(results) + output = root / args.output + output.write_text(text) + sys.stdout.write(text) + + +if __name__ == "__main__": + main()