Upgrade similarity signal to 64-bit SimHash
ober
f49910416025c0e92245d2eae1c41f5e30c41f7d
--- a/GAPS.md +++ b/GAPS.md @@ -144,6 +144,10 @@ Acceptance criteria: `sniff` uses 64-bit SimHash. This project uses a stable 32-bit FNV-derived SimHash, which is less discriminating. +Status: implemented with stable 64-bit FNV-1a-derived SimHash, deterministic +repeat-scan fixture coverage, and separate same-author vs cross-author +near-match evidence. + Acceptance criteria: - Upgrade to a stable 64-bit SimHash or document why 32-bit is retained. --- a/main-binary.ss +++ b/main-binary.ss @@ -1,7 +1,7 @@ (import (jerboa prelude)) (import (std misc process)) -(def detector-version "0.1.0") +(def detector-version "0.2.0") (def current-parent-diff-mode "first-parent") (def (use-first-parent-diffs!) @@ -550,21 +550,25 @@ evidence "large generated files, vendored updates, formatters, and mechanical refactors can match these patterns"))) -(def (fnv-step h ch) (modulo (* (bitwise-xor h (char->integer ch)) 16777619) 4294967296)) -(def (stable-hash32 s) (for/fold ([h 2166136261]) ([ch (in-string s)]) (fnv-step h ch))) +(def simhash-bits 64) +(def fnv64-offset 14695981039346656037) +(def fnv64-prime 1099511628211) +(def fnv64-modulus 18446744073709551616) +(def (fnv64-step h ch) (modulo (* (bitwise-xor h (char->integer ch)) fnv64-prime) fnv64-modulus)) +(def (stable-hash64 s) (for/fold ([h fnv64-offset]) ([ch (in-string s)]) (fnv64-step h ch))) (def (bit-set? n bit) (not (= 0 (bitwise-and n (ash 1 bit))))) -(def (simhash32 lexemes) - (let ([weights (make-vector 32 0)]) +(def (simhash64 lexemes) + (let ([weights (make-vector simhash-bits 0)]) (for ([lexeme lexemes]) - (let ([h (stable-hash32 lexeme)]) - (for ([i (in-range 32)]) + (let ([h (stable-hash64 lexeme)]) + (for ([i (in-range simhash-bits)]) (vector-set! weights i (+ (vector-ref weights i) (if (bit-set? h i) 1 -1)))))) - (for/fold ([acc 0]) ([i (in-range 32)]) + (for/fold ([acc 0]) ([i (in-range simhash-bits)]) (if (> (vector-ref weights i) 0) (bitwise-ior acc (ash 1 i)) acc)))) -(def (popcount32 n) (let loop ([x n] [acc 0]) (if (= x 0) acc (loop (ash x -1) (+ acc (bitwise-and x 1)))))) -(def (hamming32 a b) (popcount32 (bitwise-xor a b))) +(def (popcount n) (let loop ([x n] [acc 0]) (if (= x 0) acc (loop (ash x -1) (+ acc (bitwise-and x 1)))))) +(def (hamming64 a b) (popcount (bitwise-xor a b))) (def (hash-string s) - (str (stable-hash32 s))) + (str (stable-hash64 s))) (def (default-scan-config) (make-scan-config "default" 0.20 0.50 0.10 0.35 0.15 0.15 0.15 0.00 500 20000 50000)) @@ -728,16 +732,32 @@ (make-signal (signal-name s) (signal-category s) (signal-score s) (weight-for-category cfg (signal-category s) (signal-weight s)) (signal-confidence s) (signal-reason s) (signal-evidence s) (signal-limitations s))) -(def (best-sim hash hashes) - (if (null? hashes) 0.0 (- 1.0 (/ (apply min (map (lambda (h) (hamming32 hash h)) hashes)) 32.0)))) -(def (similarity-signal lines hashes) - (let* ([hash (simhash32 (source-lexemes lines))] - [sim (best-sim hash hashes)] - [score (cond [(>= sim 0.92) 0.80] [(>= sim 0.85) 0.50] [(>= sim 0.80) 0.25] [else 0.0])] - [evidence (if (> score 0.0) (list (str "best simhash similarity " sim)) '())]) +(def (simhash-entry hash author-id) (cons hash author-id)) +(def (simhash-entry-hash entry) (car entry)) +(def (simhash-entry-author entry) (cdr entry)) +(def (best-simhash64 hash entries) + (if (null? entries) + 0.0 + (- 1.0 (/ (apply min (map (lambda (entry) (hamming64 hash (simhash-entry-hash entry))) entries)) 64.0)))) +(def (similarity-score sim) + (cond [(>= sim 0.92) 0.80] [(>= sim 0.85) 0.50] [(>= sim 0.80) 0.25] [else 0.0])) +(def (similarity-signal lines hash-entries author-id) + (let* ([hash (simhash64 (source-lexemes lines))] + [same-author (filter (lambda (entry) (same-public-string? (simhash-entry-author entry) author-id)) hash-entries)] + [cross-author (filter (lambda (entry) (not (same-public-string? (simhash-entry-author entry) author-id))) hash-entries)] + [same-sim (best-simhash64 hash same-author)] + [cross-sim (best-simhash64 hash cross-author)] + [sim (max same-sim cross-sim)] + [score (similarity-score sim)] + [evidence (append (if (> (similarity-score same-sim) 0.0) + (list (str "best same-author simhash64 similarity " same-sim)) + '()) + (if (> (similarity-score cross-sim) 0.0) + (list (str "best cross-author simhash64 similarity " cross-sim)) + '()))]) (list (sig "simhash-similarity" "similarity" score 0.15 "medium" "added code is near another scanned addition" evidence "similarity in one scan window cannot prove original authorship") - hash))) + (simhash-entry hash author-id)))) (def (history-signal additions commit-time parent-time author-events) (let* ([delta (- commit-time parent-time)] @@ -891,7 +911,7 @@ [attribution (note-attributions note-obj)] [metadata (metadata-hits author-name author-email subject body note)] [eligible? (and (pair? files) (pair? lines) (or (= min-lines 0) (>= (length lines) min-lines)))] - [sim-pair (similarity-signal lines hashes)] + [sim-pair (similarity-signal lines hashes author-email)] [raw-signals (if eligible? (list (message-signal subject body adds) (code-signal lines) (cadence-code-pattern-signal lines) (structure-signal paths adds dels lines) --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -8,7 +8,8 @@ provider_tmp=$(mktemp -d) shape_fixture=$(mktemp -d) timing_fixture=$(mktemp -d) normal_timing_fixture=$(mktemp -d) -trap 'rm -rf "$fixture" "$shallow" "$provider_tmp" "$shape_fixture" "$timing_fixture" "$normal_timing_fixture"' EXIT +similarity_fixture=$(mktemp -d) +trap 'rm -rf "$fixture" "$shallow" "$provider_tmp" "$shape_fixture" "$timing_fixture" "$normal_timing_fixture" "$similarity_fixture"' EXIT git -C "$fixture" init -q git -C "$fixture" config user.name "Human Dev" @@ -176,6 +177,51 @@ if printf '%s\n' "$normal_json" | grep -q 'line velocity'; then exit 1 fi +git -C "$similarity_fixture" init -q +git -C "$similarity_fixture" config user.name "Similarity Same" +git -C "$similarity_fixture" config user.email "same@example.test" +printf 'base\n' > "$similarity_fixture/base.txt" +git -C "$similarity_fixture" add base.txt +GIT_AUTHOR_DATE='2026-07-29T12:00:00-06:00' \ +GIT_COMMITTER_DATE='2026-07-29T12:00:00-06:00' \ + git -C "$similarity_fixture" commit -q -m 'base similarity fixture' +i=1 +while [ "$i" -le 30 ]; do + printf 'def repeated_helper(value): return value + 1\n' >> "$similarity_fixture/same-a.py" + i=$((i + 1)) +done +git -C "$similarity_fixture" add same-a.py +GIT_AUTHOR_DATE='2026-07-29T12:01:00-06:00' \ +GIT_COMMITTER_DATE='2026-07-29T12:01:00-06:00' \ + git -C "$similarity_fixture" commit -q -m 'same author repeated code a' +git -C "$similarity_fixture" config user.name "Similarity Other" +git -C "$similarity_fixture" config user.email "other@example.test" +i=1 +while [ "$i" -le 30 ]; do + printf 'def repeated_helper(value): return value + 1\n' >> "$similarity_fixture/cross.py" + i=$((i + 1)) +done +git -C "$similarity_fixture" add cross.py +GIT_AUTHOR_DATE='2026-07-29T12:02:00-06:00' \ +GIT_COMMITTER_DATE='2026-07-29T12:02:00-06:00' \ + git -C "$similarity_fixture" commit -q -m 'cross author repeated code' +git -C "$similarity_fixture" config user.name "Similarity Same" +git -C "$similarity_fixture" config user.email "same@example.test" +i=1 +while [ "$i" -le 30 ]; do + printf 'def repeated_helper(value): return value + 1\n' >> "$similarity_fixture/same-b.py" + i=$((i + 1)) +done +git -C "$similarity_fixture" add same-b.py +GIT_AUTHOR_DATE='2026-07-29T12:03:00-06:00' \ +GIT_COMMITTER_DATE='2026-07-29T12:03:00-06:00' \ + git -C "$similarity_fixture" commit -q -m 'same author repeated code b' +similarity_json_one=$("$root/bin/jerboa-aigit" scan "$similarity_fixture" --format json --count 3) +similarity_json_two=$("$root/bin/jerboa-aigit" scan "$similarity_fixture" --format json --count 3) +[ "$similarity_json_one" = "$similarity_json_two" ] +printf '%s\n' "$similarity_json_one" | grep -q 'best same-author simhash64 similarity' +printf '%s\n' "$similarity_json_one" | grep -q 'best cross-author simhash64 similarity' + heuristics=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --heuristics-only) printf '%s\n' "$heuristics" | grep -q '"recorded_ai_note_present":true' if printf '%s\n' "$heuristics" | grep -q '"verdict":"recorded-ai-authorship"'; then