Add golden JSON snapshot test
ober
a1983f87de9113078268f6e4a0194ac80f644a69
--- a/GAPS.md +++ b/GAPS.md @@ -490,6 +490,11 @@ Acceptance criteria: - Avoid timestamp/path nondeterminism in snapshots. - Review intentional output contract changes via snapshot diffs. +Status: implemented with `tests/golden-json.sh` and +`tests/golden/scan-basic.json`. The snapshot normalizes temporary repository +paths, uses fixed commit dates and sorted JSON keys, supports +`UPDATE_GOLDEN=1`, and is part of `make test`/`make verify`. + ### G-052: No property/fuzz tests The plan calls for arbitrary text, score bounds, deterministic output, bounded --- a/Makefile +++ b/Makefile @@ -19,10 +19,12 @@ $(BIN): bin/jerboa-aigit test: @tests/fixture-smoke.sh + @tests/golden-json.sh verify: @$(JERBOA) main-binary.ss --help >/dev/null @tests/fixture-smoke.sh + @tests/golden-json.sh binary: @mkdir -p dist new file mode 100755 --- /dev/null +++ b/tests/golden-json.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env sh +set -eu + +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +fixture=$(mktemp -d) +out=$(mktemp) +trap 'rm -rf "$fixture"; rm -f "$out"' EXIT + +git -C "$fixture" init -q +git -C "$fixture" config user.name "Golden Human" +git -C "$fixture" config user.email "golden-human@example.test" +printf 'baseline\n' > "$fixture/README.md" +git -C "$fixture" add README.md +GIT_AUTHOR_DATE='2026-07-29T10:00:00-06:00' \ +GIT_COMMITTER_DATE='2026-07-29T10:00:00-06:00' \ + git -C "$fixture" commit -q -m 'baseline' + +git -C "$fixture" config user.name "Codex" +git -C "$fixture" config user.email "codex@openai.example" +mkdir -p "$fixture/src" +i=1 +while [ "$i" -le 12 ]; do + { + printf 'def golden_helper_%s(value):\n' "$i" + printf ' """This function ensures robust handling of the provided value."""\n' + printf ' result = value + %s\n' "$i" + printf ' return result\n\n' + } >> "$fixture/src/golden.py" + i=$((i + 1)) +done +git -C "$fixture" add src/golden.py +GIT_AUTHOR_DATE='2026-07-29T10:01:00-06:00' \ +GIT_COMMITTER_DATE='2026-07-29T10:01:00-06:00' \ + git -C "$fixture" commit -q -m 'feat: Implement golden helpers.' \ + -m 'This commit introduces a comprehensive helper implementation designed to ensure that users can process values.' +git -C "$fixture" notes --ref=ai add -m '{"tool":"codex","model":"gpt-5","lines":[{"path":"src/golden.py","start":1,"end":48}]}' HEAD + +"$root/bin/jerboa-aigit" scan "$fixture" --format json --count 2 > "$out.raw" +python3 - "$fixture" "$out.raw" "$out" <<'PY' +import json +import os +import sys + +repo = sys.argv[1] +raw = sys.argv[2] +out = sys.argv[3] +repo_variants = {repo, os.path.realpath(repo)} + +with open(raw, "r", encoding="utf-8") as f: + data = json.load(f) + +def normalize(value): + if isinstance(value, str): + return "<repo>" if value in repo_variants else value + if isinstance(value, list): + return [normalize(item) for item in value] + if isinstance(value, dict): + return {key: normalize(value[key]) for key in sorted(value)} + return value + +with open(out, "w", encoding="utf-8") as f: + json.dump(normalize(data), f, indent=2, sort_keys=True) + f.write("\n") +PY + +golden="$root/tests/golden/scan-basic.json" +if [ "${UPDATE_GOLDEN:-0}" = 1 ]; then + mkdir -p "$root/tests/golden" + cp "$out" "$golden" + echo "updated $golden" +else + diff -u "$golden" "$out" + echo "golden JSON snapshots passed" +fi new file mode 100644 --- /dev/null +++ b/tests/golden/scan-basic.json @@ -0,0 +1,431 @@ +{ + "analysis_provider": "offline-heuristics", + "config_hash": "17715209077406632572", + "count": 2, + "detector_version": "0.2.0", + "embeddings_used": false, + "findings": [ + { + "added_lines": 60, + "additions": 60, + "analysis_provider": "offline-heuristics", + "author_email": "codex@openai.example", + "author_name": "Codex", + "commit": "d696d34431e81654c36e517e39af96bb1f0e8caa", + "config_hash": "17715209077406632572", + "deletions": 0, + "detector_version": "0.2.0", + "embeddings_used": false, + "file_findings": [ + { + "attribution_status": "matched-note-range", + "commit_score": 0.9344999999999999, + "commit_signal_categories": [ + "text", + "code", + "structure", + "semantic", + "similarity", + "history", + "baseline" + ], + "commit_verdict": "recorded-ai-authorship", + "path": "src/golden.py", + "recorded_ai_lines": 48, + "recorded_attribution": [ + { + "end": 48, + "model": "gpt-5", + "path": "src/golden.py", + "session": "", + "source": "refs/notes/ai", + "start": 1, + "tool": "codex" + } + ] + } + ], + "files": [ + "src/golden.py" + ], + "llm_used": false, + "metadata_hits": [ + "codex", + "openai" + ], + "network_used": false, + "parent": "b2a3d4a37749db898734dc9e5e381d1d7a9ea10c", + "provider": "none", + "recorded_ai_note_excerpt": "{\"tool\":\"codex\",\"model\":\"gpt-5\",\"lines\":[{\"path\":\"src/golden.py\",\"start\":1,\"end\":48}]}", + "recorded_ai_note_present": true, + "recorded_attribution": [ + { + "end": 48, + "model": "gpt-5", + "path": "src/golden.py", + "session": "", + "source": "refs/notes/ai", + "start": 1, + "tool": "codex" + } + ], + "recovered_attribution": [ + { + "agent": "codex", + "confidence": 0.82, + "evidence": "known agent identity matched author name or email", + "recorded_provenance": false, + "source": "author_identity" + }, + { + "agent": "openai", + "confidence": 0.82, + "evidence": "known agent identity matched author name or email", + "recorded_provenance": false, + "source": "author_identity" + }, + { + "agent": "codex", + "confidence": 0.66, + "evidence": "known agent identity matched refs/notes/ai metadata", + "recorded_provenance": false, + "source": "recorded_note_metadata" + } + ], + "repository": "<repo>", + "score": 0.9344999999999999, + "signals": [ + { + "category": "text", + "confidence": "low", + "evidence": [ + "conventional prefix", + "this commit introduces", + "comprehensive", + "ensure that", + "designed to" + ], + "limitations": "message style alone is weak evidence", + "name": "message-style", + "reason": "commit message matches weak generated-text patterns", + "score": 0.42, + "weight": 0.1 + }, + { + "category": "text", + "confidence": "low", + "evidence": [ + "formulaic generated-message phrase count 4" + ], + "limitations": "weak offline approximation; not GPT-2 perplexity and not proof of AI authorship", + "name": "message-naturalness", + "reason": "offline message naturalness proxy found formulaic or unusually smooth text", + "score": 0.12, + "weight": 0.1 + }, + { + "category": "code", + "confidence": "medium", + "evidence": [ + "generic identifier ratio 0.3076923076923077", + "this function", + "ensure" + ], + "limitations": "language-neutral lexical analysis is noisy", + "name": "lexical-code-style", + "reason": "added code has lexical/comment patterns associated with generated scaffolding", + "score": 0.31, + "weight": 0.35 + }, + { + "category": "code", + "confidence": "medium", + "evidence": [ + "regular generated-style names 12", + "large scaffold-like block" + ], + "limitations": "framework conventions, generated code, tutorials, and mechanical scaffolds can match these patterns", + "name": "cadence-code-patterns", + "reason": "Cadence-style code pattern strategies matched generic names, regular naming, or scaffolding", + "score": 0.32, + "weight": 0.35 + }, + { + "category": "structure", + "confidence": "medium", + "evidence": [ + "regular blank-line spacing" + ], + "limitations": "formatters, generated files, and refactors can look similar", + "name": "diff-structure", + "reason": "diff shape is unusually regular or broad", + "score": 0.2, + "weight": 0.15 + }, + { + "category": "structure", + "confidence": "medium", + "evidence": [], + "limitations": "large generated files, vendored updates, formatters, and mechanical refactors can match these patterns", + "name": "cadence-diff-shape", + "reason": "Cadence-style diff shape strategies matched broad or regular changes", + "score": 0.0, + "weight": 0.15 + }, + { + "category": "semantic", + "confidence": "low", + "evidence": [], + "limitations": "lexical overlap is a weak offline proxy for semantic self-summary and can match intentional human summaries", + "name": "semantic-alignment", + "reason": "commit message vocabulary overlaps unusually strongly with added code", + "score": 0.0, + "weight": 0.1 + }, + { + "category": "similarity", + "confidence": "medium", + "evidence": [], + "limitations": "similarity in one scan window cannot prove original authorship", + "name": "simhash-similarity", + "reason": "added code is near another scanned addition", + "score": 0.0, + "weight": 0.15 + }, + { + "category": "history", + "confidence": "low", + "evidence": [ + "line velocity 60.0 additions/minute" + ], + "limitations": "Git commit timestamps are not typing timestamps", + "name": "history-timing", + "reason": "commit timestamps suggest high throughput or bursts", + "score": 1.0, + "weight": 0.15 + }, + { + "category": "baseline", + "confidence": "low", + "evidence": [ + "fewer than five comparison commits" + ], + "limitations": "missing baseline is not evidence of human authorship", + "name": "author-baseline", + "reason": "author baseline unavailable", + "score": 0.0, + "weight": 0.0 + }, + { + "category": "baseline", + "confidence": "low", + "evidence": [ + "fewer than five comparison commits" + ], + "limitations": "missing repository baseline is not evidence of human authorship", + "name": "repository-baseline", + "reason": "repository baseline unavailable", + "score": 0.0, + "weight": 0.0 + }, + { + "category": "code", + "confidence": "medium", + "evidence": [ + "generated-style Python docstrings 12", + "repetitive Python function structure 12 defs / 12 returns", + "regular Python function length gaps stddev 0.0", + "low Python identifier diversity 0.14619883040935672" + ], + "limitations": "dependency-free textual Python analysis approximates AST features and can match generated or repetitive human code", + "name": "python-ast-style", + "reason": "Python-specific docstring, function-structure, and identifier patterns matched", + "score": 0.8, + "weight": 0.35 + } + ], + "subject": "feat: Implement golden helpers.", + "time": 1785340860, + "unmatched_recorded_attribution": [], + "verdict": "recorded-ai-authorship", + "warnings": [] + }, + { + "added_lines": 1, + "additions": 1, + "analysis_provider": "offline-heuristics", + "author_email": "golden-human@example.test", + "author_name": "Golden Human", + "commit": "b2a3d4a37749db898734dc9e5e381d1d7a9ea10c", + "config_hash": "17715209077406632572", + "deletions": 0, + "detector_version": "0.2.0", + "embeddings_used": false, + "file_findings": [ + { + "attribution_status": "no-recorded-range", + "commit_score": 0.060000000000000005, + "commit_signal_categories": [ + "text", + "code", + "structure", + "semantic", + "similarity", + "history", + "baseline" + ], + "commit_verdict": "likely-human-style", + "path": "README.md", + "recorded_ai_lines": 0, + "recorded_attribution": [] + } + ], + "files": [ + "README.md" + ], + "llm_used": false, + "metadata_hits": [], + "network_used": false, + "parent": "", + "provider": "none", + "recorded_ai_note_excerpt": "", + "recorded_ai_note_present": false, + "recorded_attribution": [], + "recovered_attribution": [], + "repository": "<repo>", + "score": 0.060000000000000005, + "signals": [ + { + "category": "text", + "confidence": "low", + "evidence": [ + "very terse subject" + ], + "limitations": "message style alone is weak evidence", + "name": "message-style", + "reason": "commit message matches weak generated-text patterns", + "score": 0.15, + "weight": 0.1 + }, + { + "category": "text", + "confidence": "low", + "evidence": [], + "limitations": "weak offline approximation; not GPT-2 perplexity and not proof of AI authorship", + "name": "message-naturalness", + "reason": "offline message naturalness proxy found formulaic or unusually smooth text", + "score": 0.0, + "weight": 0.1 + }, + { + "category": "code", + "confidence": "medium", + "evidence": [], + "limitations": "language-neutral lexical analysis is noisy", + "name": "lexical-code-style", + "reason": "added code has lexical/comment patterns associated with generated scaffolding", + "score": 0.0, + "weight": 0.35 + }, + { + "category": "code", + "confidence": "medium", + "evidence": [], + "limitations": "framework conventions, generated code, tutorials, and mechanical scaffolds can match these patterns", + "name": "cadence-code-patterns", + "reason": "Cadence-style code pattern strategies matched generic names, regular naming, or scaffolding", + "score": 0.0, + "weight": 0.35 + }, + { + "category": "structure", + "confidence": "medium", + "evidence": [], + "limitations": "formatters, generated files, and refactors can look similar", + "name": "diff-structure", + "reason": "diff shape is unusually regular or broad", + "score": 0.0, + "weight": 0.15 + }, + { + "category": "structure", + "confidence": "medium", + "evidence": [], + "limitations": "large generated files, vendored updates, formatters, and mechanical refactors can match these patterns", + "name": "cadence-diff-shape", + "reason": "Cadence-style diff shape strategies matched broad or regular changes", + "score": 0.0, + "weight": 0.15 + }, + { + "category": "semantic", + "confidence": "low", + "evidence": [ + "message/code term overlap 1.0: baseline" + ], + "limitations": "lexical overlap is a weak offline proxy for semantic self-summary and can match intentional human summaries", + "name": "semantic-alignment", + "reason": "commit message vocabulary overlaps unusually strongly with added code", + "score": 0.45, + "weight": 0.1 + }, + { + "category": "similarity", + "confidence": "medium", + "evidence": [], + "limitations": "similarity in one scan window cannot prove original authorship", + "name": "simhash-similarity", + "reason": "added code is near another scanned addition", + "score": 0.0, + "weight": 0.15 + }, + { + "category": "history", + "confidence": "low", + "evidence": [], + "limitations": "Git commit timestamps are not typing timestamps", + "name": "history-timing", + "reason": "commit timestamps suggest high throughput or bursts", + "score": 0.0, + "weight": 0.15 + }, + { + "category": "baseline", + "confidence": "low", + "evidence": [ + "fewer than five comparison commits" + ], + "limitations": "missing baseline is not evidence of human authorship", + "name": "author-baseline", + "reason": "author baseline unavailable", + "score": 0.0, + "weight": 0.0 + }, + { + "category": "baseline", + "confidence": "low", + "evidence": [ + "fewer than five comparison commits" + ], + "limitations": "missing repository baseline is not evidence of human authorship", + "name": "repository-baseline", + "reason": "repository baseline unavailable", + "score": 0.0, + "weight": 0.0 + } + ], + "subject": "baseline", + "time": 1785340800, + "unmatched_recorded_attribution": [], + "verdict": "likely-human-style", + "warnings": [ + "root commit; no parent diff baseline", + "no refs/notes/ai authorship note found" + ] + } + ], + "llm_used": false, + "network_used": false, + "provider": "none", + "repository": "<repo>" +}