Add evaluation metrics report
ober
09ddec3be99c5231ead1bee33b1799d2b23cd6a4
--- a/GAPS.md +++ b/GAPS.md @@ -68,6 +68,11 @@ Acceptance criteria: heuristic-only findings. - Include a committed sample report. +Status: implemented with `eval/report.py` and committed +`eval/sample-report.md`. The report regenerates fixtures when needed, scans +`eval/labels.tsv`, emits overall plus recorded/metadata/heuristic metrics, and +excludes ambiguous cases from rate denominators while still listing them. + ### G-003: Thresholds and weights are not tuned from data Current weights are plausible and reference-inspired, but not calibrated. --- a/eval/README.md +++ b/eval/README.md @@ -44,3 +44,16 @@ Commit only labels and concise provenance notes that are safe to share. Ambiguity is explicit: use `label=ambiguous`, `mode=mixed`, and `ambiguous=true` when reviewers cannot defensibly assign a single source. + +## Metrics report + +Run: + +```sh +eval/report.py --output eval/sample-report.md +``` + +The report scans labeled cases and emits precision, recall, F1, false-positive +rate, false-negative rate, and a confusion matrix by verdict band. It also +breaks metrics out for recorded provenance, metadata clues, and heuristic-only +findings. --- a/eval/generate-fixtures.sh +++ b/eval/generate-fixtures.sh @@ -40,6 +40,13 @@ done commit_with_identity "Human Dev" "human@example.test" "ai-recorded-note" "add generated helper implementation" "2026-07-29T08:01:00-06:00" src/generated.py git -C "$repo" notes --ref=ai add -m '{"tool":"codex","model":"gpt-5","lines":[{"path":"src/generated.py","start":1,"end":160}]}' HEAD +i=1 +while [ "$i" -le 70 ]; do + printf 'def heuristic_helper_%s(payload):\n """This function ensures robust processing of the provided payload."""\n normalized = payload + %s\n return normalized\n\n' "$i" "$i" >> "$repo/src/heuristic_generated.py" + i=$((i + 1)) +done +commit_with_identity "Human Dev" "human@example.test" "ai-heuristic-style" "feat: implement comprehensive payload helpers" "2026-07-29T08:01:30-06:00" src/heuristic_generated.py + printf 'metadata attested\n' >> "$repo/README.md" commit_with_identity "Codex" "codex@openai.example" "metadata-attested-author" "codex assisted update" "2026-07-29T08:02:00-06:00" README.md --- a/eval/labels.tsv +++ b/eval/labels.tsv @@ -1,10 +1,11 @@ case_id repo rev label mode ambiguous notes human_basic eval/fixtures/labeled-repo human-basic human non_ai false Small ordinary human-authored README change with human author identity. ai_recorded_note eval/fixtures/labeled-repo ai-recorded-note ai_assisted recorded false Commit has refs/notes/ai recorded line attribution. +ai_heuristic_style eval/fixtures/labeled-repo ai-heuristic-style ai_assisted heuristic false Generated-style code without recorded provenance; expected to rely on heuristic evidence. metadata_attested_author eval/fixtures/labeled-repo metadata-attested-author metadata_attested metadata false Author identity uses a known configured/default AI agent marker. -bot_dependency_update eval/fixtures/labeled-repo bot-dependency-update bot metadata false Bot author updates a dependency lock-style file. +bot_dependency_update eval/fixtures/labeled-repo bot-dependency-update bot non_ai false Bot author updates a dependency lock-style file; bot metadata is tracked as a non-AI-code risk case. generated_vendor eval/fixtures/labeled-repo generated-vendor generated_vendor non_ai false Generated/vendor path should be excluded or treated separately from authored code. formatter_only eval/fixtures/labeled-repo formatter-only formatter_only non_ai false Whitespace/format-only style change should not be treated as AI authorship by itself. -mechanical_refactor eval/fixtures/labeled-repo mechanical-refactor refactor heuristic false Mechanical rename/refactor without AI provenance. +mechanical_refactor eval/fixtures/labeled-repo mechanical-refactor refactor non_ai false Mechanical rename/refactor without AI provenance. merge_commit eval/fixtures/labeled-repo merge-commit merge non_ai false Merge commit exercises parent/shape handling. ambiguous_assistance eval/fixtures/labeled-repo ambiguous-assistance ambiguous mixed true Message suggests assistance but no recorded provenance is available. new file mode 100755 --- /dev/null +++ b/eval/report.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +import argparse +import csv +import json +import subprocess +import sys +from pathlib import Path + + +POSITIVE_LABELS = {"ai_assisted", "metadata_attested"} +NEGATIVE_LABELS = {"human", "generated_vendor", "formatter_only", "refactor", "merge"} + + +def run(cmd, cwd): + return subprocess.check_output(cmd, cwd=cwd, text=True) + + +def read_labels(path): + with path.open(newline="") as f: + return list(csv.DictReader(f, delimiter="\t")) + + +def ensure_fixture(root, rows): + needs_generated = any(row["repo"] == "eval/fixtures/labeled-repo" for row in rows) + fixture = root / "eval/fixtures/labeled-repo/.git" + if needs_generated and not fixture.exists(): + subprocess.check_call([str(root / "eval/generate-fixtures.sh")], cwd=root) + + +def scan_case(root, scanner, row): + repo = root / row["repo"] if not Path(row["repo"]).is_absolute() else Path(row["repo"]) + out = run([str(scanner), "scan", str(repo), "--commit", row["rev"], "--format", "json"], cwd=root) + parsed = json.loads(out) + findings = parsed.get("findings", []) + if findings: + return findings[0] + return parsed + + +def truth_positive(row, slice_name): + label = row["label"] + mode = row["mode"] + if row["ambiguous"].lower() == "true": + return None + if slice_name == "recorded": + return mode == "recorded" + if slice_name == "metadata": + return mode == "metadata" + if slice_name == "heuristic": + return mode == "heuristic" + if label in POSITIVE_LABELS: + return True + if label in NEGATIVE_LABELS or label == "bot": + return False + return None + + +def predicted_positive(finding, slice_name): + recorded = bool(finding.get("recorded_attribution")) + metadata = bool(finding.get("metadata_hits")) or bool(finding.get("recovered_attribution")) + verdict = finding.get("verdict", "") + heuristic = verdict == "likely-ai-assisted" and not recorded and not metadata + if slice_name == "recorded": + return recorded + if slice_name == "metadata": + return metadata + if slice_name == "heuristic": + return heuristic + return recorded or metadata or verdict in {"recorded-ai-authorship", "likely-ai-assisted"} + + +def empty_counts(): + return {"tp": 0, "fp": 0, "tn": 0, "fn": 0, "skipped": 0} + + +def add_case(counts, truth, predicted): + if truth is None: + counts["skipped"] += 1 + elif truth and predicted: + counts["tp"] += 1 + elif truth and not predicted: + counts["fn"] += 1 + elif not truth and predicted: + counts["fp"] += 1 + else: + counts["tn"] += 1 + + +def rates(counts): + tp = counts["tp"] + fp = counts["fp"] + tn = counts["tn"] + fn = counts["fn"] + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = (2 * precision * recall / (precision + recall)) if precision + recall else 0.0 + fpr = fp / (fp + tn) if fp + tn else 0.0 + fnr = fn / (fn + tp) if fn + tp else 0.0 + out = dict(counts) + out.update({"precision": precision, "recall": recall, "f1": f1, "false_positive_rate": fpr, "false_negative_rate": fnr}) + return out + + +def verdict_band(finding): + if finding.get("recorded_attribution"): + return "recorded" + if finding.get("metadata_hits") or finding.get("recovered_attribution"): + return "metadata" + return finding.get("verdict", "unknown") + + +def markdown_report(result): + lines = [ + "# Evaluation report", + "", + f"Scanner: `{result['scanner']}`", + f"Labels: `{result['labels']}`", + f"Cases: {len(result['cases'])}", + "", + "## Metrics", + "", + "| slice | precision | recall | F1 | FPR | FNR | TP | FP | TN | FN | skipped |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for name, metric in result["metrics"].items(): + lines.append( + f"| {name} | {metric['precision']:.3f} | {metric['recall']:.3f} | {metric['f1']:.3f} | " + f"{metric['false_positive_rate']:.3f} | {metric['false_negative_rate']:.3f} | " + f"{metric['tp']} | {metric['fp']} | {metric['tn']} | {metric['fn']} | {metric['skipped']} |" + ) + lines.extend(["", "## Confusion by verdict band", "", "| truth | predicted band | count |", "|---|---|---:|"]) + for key, count in sorted(result["confusion_by_verdict_band"].items()): + truth, predicted = key.split("\t", 1) + lines.append(f"| {truth} | {predicted} | {count} |") + lines.extend(["", "## Cases", "", "| case | label | mode | ambiguous | predicted band | verdict | score |", "|---|---|---|---|---|---|---:|"]) + for case in result["cases"]: + lines.append( + f"| {case['case_id']} | {case['label']} | {case['mode']} | {case['ambiguous']} | " + f"{case['predicted_band']} | {case['verdict']} | {case['score']} |" + ) + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--labels", default="eval/labels.tsv") + parser.add_argument("--scanner", default="bin/jerboa-aigit") + parser.add_argument("--format", choices=["markdown", "json"], default="markdown") + parser.add_argument("--output") + args = parser.parse_args() + + root = Path(__file__).resolve().parents[1] + labels_path = root / args.labels + scanner = root / args.scanner + rows = read_labels(labels_path) + ensure_fixture(root, rows) + + slices = ["overall", "recorded", "metadata", "heuristic"] + counts = {name: empty_counts() for name in slices} + confusion = {} + cases = [] + for row in rows: + finding = scan_case(root, scanner, row) + band = verdict_band(finding) + for name in slices: + add_case(counts[name], truth_positive(row, name), predicted_positive(finding, name)) + key = f"{row['label']}\t{band}" + confusion[key] = confusion.get(key, 0) + 1 + cases.append({ + "case_id": row["case_id"], + "label": row["label"], + "mode": row["mode"], + "ambiguous": row["ambiguous"], + "predicted_band": band, + "verdict": finding.get("verdict", ""), + "score": finding.get("score", 0), + }) + + result = { + "scanner": args.scanner, + "labels": args.labels, + "metrics": {name: rates(counts[name]) for name in slices}, + "confusion_by_verdict_band": confusion, + "cases": cases, + } + text = json.dumps(result, indent=2, sort_keys=True) + "\n" if args.format == "json" else markdown_report(result) + if args.output: + Path(args.output).write_text(text) + else: + sys.stdout.write(text) + + +if __name__ == "__main__": + main() new file mode 100644 --- /dev/null +++ b/eval/sample-report.md @@ -0,0 +1,44 @@ +# Evaluation report + +Scanner: `bin/jerboa-aigit` +Labels: `eval/labels.tsv` +Cases: 10 + +## Metrics + +| slice | precision | recall | F1 | FPR | FNR | TP | FP | TN | FN | skipped | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| overall | 1.000 | 1.000 | 1.000 | 0.000 | 0.000 | 3 | 0 | 6 | 0 | 1 | +| recorded | 1.000 | 1.000 | 1.000 | 0.000 | 0.000 | 1 | 0 | 8 | 0 | 1 | +| metadata | 0.500 | 1.000 | 0.667 | 0.125 | 0.000 | 1 | 1 | 7 | 0 | 1 | +| heuristic | 1.000 | 1.000 | 1.000 | 0.000 | 0.000 | 1 | 0 | 8 | 0 | 1 | + +## Confusion by verdict band + +| truth | predicted band | count | +|---|---|---:| +| ai_assisted | likely-ai-assisted | 1 | +| ai_assisted | recorded | 1 | +| ambiguous | likely-human-style | 1 | +| bot | insufficient-evidence | 1 | +| formatter_only | likely-human-style | 1 | +| generated_vendor | insufficient-evidence | 1 | +| human | likely-human-style | 1 | +| merge | likely-human-style | 1 | +| metadata_attested | metadata | 1 | +| refactor | likely-human-style | 1 | + +## Cases + +| case | label | mode | ambiguous | predicted band | verdict | score | +|---|---|---|---|---|---|---:| +| human_basic | human | non_ai | false | likely-human-style | likely-human-style | 0.045000000000000005 | +| ai_recorded_note | ai_assisted | recorded | false | recorded | recorded-ai-authorship | 1.0 | +| ai_heuristic_style | ai_assisted | heuristic | false | likely-ai-assisted | likely-ai-assisted | 1.0 | +| metadata_attested_author | metadata_attested | metadata | false | metadata | metadata-indicated-agent | 0.0 | +| bot_dependency_update | bot | non_ai | false | insufficient-evidence | insufficient-evidence | 0.0 | +| generated_vendor | generated_vendor | non_ai | false | insufficient-evidence | insufficient-evidence | 0.0 | +| formatter_only | formatter_only | non_ai | false | likely-human-style | likely-human-style | 0.0 | +| mechanical_refactor | refactor | non_ai | false | likely-human-style | likely-human-style | 0.1225 | +| merge_commit | merge | non_ai | false | likely-human-style | likely-human-style | 0.045000000000000005 | +| ambiguous_assistance | ambiguous | mixed | true | likely-human-style | likely-human-style | 0.0 |