Add reference differential report
ober
5b638612d9a490ff24f5fc0f94cdf7bdd789d940
--- a/GAPS.md +++ b/GAPS.md @@ -102,6 +102,13 @@ Acceptance criteria: about provenance or safety. - Preserve raw reference outputs or concise normalized summaries. +Status: implemented with `eval/differential.py` and committed +`eval/differential-report.md`. The report compares overlapping generated-corpus +cases with installed `sniff`, checks git-ai-compatible `refs/notes/ai` +presence, records Cadence availability via `CADENCE_BIN`, and documents +intentional differences around provenance separation, offline defaults, and +vendor/generated exclusions. + ## P1: High-value sniff parity ### G-010: Missing semantic alignment signal --- a/eval/README.md +++ b/eval/README.md @@ -67,3 +67,14 @@ eval/sensitivity.py --output eval/sensitivity-report.md This compares built-in defaults against `eval/configs/permissive.json` and `eval/configs/strict.json`, preserving each finding's config hash in the underlying evaluation report. + +Run reference differentials: + +```sh +eval/differential.py --output eval/differential-report.md +``` + +The differential report compares `jerboa-aigit` with the installed `sniff` +package when available, treats git-ai as the `refs/notes/ai` provenance +reference, and runs Cadence only when `CADENCE_BIN` points to a prebuilt +executable. new file mode 100644 --- /dev/null +++ b/eval/differential-report.md @@ -0,0 +1,23 @@ +# Differential comparison report + +This report compares overlapping generated-corpus cases against local reference tools when available. + +Intentional differences: + +- `jerboa-aigit` separates recorded provenance from recovered metadata evidence. +- `jerboa-aigit` keeps network/LLM providers opt-in and reports offline heuristic limitations. +- Vendor/generated paths are excluded by default; reference tools may count them differently. +- git-ai is treated as the refs/notes/ai provenance reference, not as a heuristic detector. + +| case | label | jerboa band | jerboa verdict | sniff band | cadence | git-ai notes | agreement notes | +|---|---|---|---|---|---|---|---| +| human_basic | human | likely-human-style | likely-human-style | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| ai_recorded_note | ai_assisted | recorded | recorded-ai-authorship | Likely AI-assisted | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | refs/notes/ai present | recorded-note agreement; sniff returned overlapping commit | +| ai_heuristic_style | ai_assisted | likely-ai-assisted | likely-ai-assisted | Likely AI-assisted | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| metadata_attested_author | metadata_attested | metadata | metadata-indicated-agent | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| bot_dependency_update | bot | insufficient-evidence | insufficient-evidence | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| generated_vendor | generated_vendor | insufficient-evidence | insufficient-evidence | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| formatter_only | formatter_only | likely-human-style | likely-human-style | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| mechanical_refactor | refactor | likely-human-style | likely-human-style | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| merge_commit | merge | likely-human-style | likely-human-style | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | +| ambiguous_assistance | ambiguous | likely-human-style | likely-human-style | Likely Human | CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout | no refs/notes/ai note | recorded-note agreement; sniff returned overlapping commit | new file mode 100755 --- /dev/null +++ b/eval/differential.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import argparse +import csv +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + + +def run(cmd, cwd): + return subprocess.check_output(cmd, cwd=cwd, text=True, stderr=subprocess.STDOUT) + + +def read_labels(path): + with path.open(newline="") as f: + return list(csv.DictReader(f, delimiter="\t")) + + +def ensure_fixture(root, rows): + needs_generated = any(row["repo"] == "eval/fixtures/labeled-repo" for row in rows) + fixture = root / "eval/fixtures/labeled-repo/.git" + if needs_generated and not fixture.exists(): + subprocess.check_call([str(root / "eval/generate-fixtures.sh")], cwd=root) + + +def repo_path(root, row): + repo = Path(row["repo"]) + return repo if repo.is_absolute() else root / repo + + +def rev_sha(repo, rev): + return run(["git", "-C", str(repo), "rev-parse", f"{rev}^{{commit}}"], cwd=repo).strip() + + +def jerboa_case(root, scanner, row): + repo = repo_path(root, row) + out = run([str(scanner), "scan", str(repo), "--commit", row["rev"], "--format", "json"], cwd=root) + parsed = json.loads(out) + findings = parsed.get("findings", []) + return findings[0] if findings else parsed + + +def sniff_python(): + explicit = os.environ.get("SNIFF_PYTHON") + if explicit: + return explicit + candidate = Path("/Users/user/.local/pipx/venvs/sniff-cli/bin/python") + return str(candidate) if candidate.exists() else "" + + +def sniff_repo(repo, count): + py = sniff_python() + if not py: + return {"available": False, "reason": "SNIFF_PYTHON not set and pipx sniff python not found", "results": []} + code = ( + "import json, sys\n" + "from sniff_cli.main import _get_analysis_data\n" + "results, err = _get_analysis_data(sys.argv[1], int(sys.argv[2]), use_llm=False)\n" + "print(json.dumps({'error': err, 'results': results or []}))\n" + ) + try: + raw = subprocess.check_output([py, "-c", code, str(repo), str(count)], cwd=repo, text=True, stderr=subprocess.DEVNULL) + data = json.loads(raw) + return {"available": data.get("error") is None, "reason": data.get("error") or "", "results": data.get("results", [])} + except Exception as exc: + return {"available": False, "reason": str(exc), "results": []} + + +def sniff_case(sniff_data, sha): + for item in sniff_data.get("results", []): + if item.get("hash") == sha: + return {"available": True, "band": item.get("band", ""), "score": item.get("score", 0), "reasons": item.get("reasons", [])} + return {"available": sniff_data.get("available", False), "band": "not-returned", "score": 0, "reasons": [sniff_data.get("reason", "")]} + + +def cadence_case(root, row): + cadence_bin = os.environ.get("CADENCE_BIN") + if not cadence_bin or not os.access(cadence_bin, os.X_OK): + return {"available": False, "summary": "CADENCE_BIN not set to an executable; skipped to avoid building/writing in sibling checkout"} + repo = repo_path(root, row) + with tempfile.TemporaryDirectory() as tmp: + name = "cadence-differential.json" + try: + run([cadence_bin, "analyze", str(repo), "--output", name], cwd=Path(tmp)) + output = Path(tmp) / "reports" / name + data = json.loads(output.read_text()) if output.exists() else {} + return {"available": True, "summary": f"detections={data.get('detection_count', data.get('DetectionCount', 'unknown'))}"} + except Exception as exc: + return {"available": False, "summary": str(exc)} + + +def git_ai_notes_case(repo, sha): + try: + note = run(["git", "-C", str(repo), "notes", "--ref=ai", "show", sha], cwd=repo) + return {"available": True, "recorded_note": bool(note.strip()), "summary": "refs/notes/ai present"} + except Exception: + return {"available": True, "recorded_note": False, "summary": "no refs/notes/ai note"} + + +def jerboa_band(finding): + if finding.get("recorded_attribution"): + return "recorded" + if finding.get("metadata_hits") or finding.get("recovered_attribution"): + return "metadata" + return finding.get("verdict", "unknown") + + +def markdown(rows): + lines = [ + "# Differential comparison report", + "", + "This report compares overlapping generated-corpus cases against local reference tools when available.", + "", + "Intentional differences:", + "", + "- `jerboa-aigit` separates recorded provenance from recovered metadata evidence.", + "- `jerboa-aigit` keeps network/LLM providers opt-in and reports offline heuristic limitations.", + "- Vendor/generated paths are excluded by default; reference tools may count them differently.", + "- git-ai is treated as the refs/notes/ai provenance reference, not as a heuristic detector.", + "", + "| case | label | jerboa band | jerboa verdict | sniff band | cadence | git-ai notes | agreement notes |", + "|---|---|---|---|---|---|---|---|", + ] + for row in rows: + lines.append( + f"| {row['case_id']} | {row['label']} | {row['jerboa_band']} | {row['jerboa_verdict']} | " + f"{row['sniff_band']} | {row['cadence_summary']} | {row['git_ai_summary']} | {row['agreement_notes']} |" + ) + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--labels", default="eval/labels.tsv") + parser.add_argument("--scanner", default="bin/jerboa-aigit") + parser.add_argument("--output", default="eval/differential-report.md") + args = parser.parse_args() + + root = Path(__file__).resolve().parents[1] + rows = read_labels(root / args.labels) + ensure_fixture(root, rows) + scanner = root / args.scanner + + sniff_cache = {} + out_rows = [] + for row in rows: + repo = repo_path(root, row) + sha = rev_sha(repo, row["rev"]) + repo_key = str(repo) + if repo_key not in sniff_cache: + sniff_cache[repo_key] = sniff_repo(repo, max(len(rows) + 20, 60)) + jerboa = jerboa_case(root, scanner, row) + sniff = sniff_case(sniff_cache[repo_key], sha) + cadence = cadence_case(root, row) + git_ai = git_ai_notes_case(repo, sha) + agreement = [] + if git_ai["recorded_note"] == bool(jerboa.get("recorded_attribution")): + agreement.append("recorded-note agreement") + else: + agreement.append("recorded-note mismatch") + if sniff.get("available") and sniff.get("band") not in {"", "not-returned"}: + agreement.append("sniff returned overlapping commit") + else: + agreement.append("sniff unavailable/not returned") + out_rows.append({ + "case_id": row["case_id"], + "label": row["label"], + "jerboa_band": jerboa_band(jerboa), + "jerboa_verdict": jerboa.get("verdict", ""), + "sniff_band": sniff.get("band", "unavailable"), + "cadence_summary": cadence.get("summary", "unavailable"), + "git_ai_summary": git_ai.get("summary", ""), + "agreement_notes": "; ".join(agreement), + }) + + text = markdown(out_rows) + output = root / args.output + output.write_text(text) + sys.stdout.write(text) + + +if __name__ == "__main__": + main()