Add offline message naturalness signal
ober
740ee94ba983941245eaeb1595e7ed39e6045f5e
--- a/GAPS.md +++ b/GAPS.md @@ -121,6 +121,11 @@ Acceptance criteria: `sniff` has optional GPT-2 perplexity. This project lacks any equivalent. +Status: implemented as `message-naturalness`, a weak offline naturalness proxy +that reports formulaic phrases, low unique-word ratio, and over-polished long +messages while explicitly documenting that it is not GPT-2 perplexity. Local +provider malformed-output handling is covered by fixtures. + Acceptance criteria: - Add a provider interface or offline approximation for message naturalness. --- a/main-binary.ss +++ b/main-binary.ss @@ -408,6 +408,44 @@ (def (source-lexemes lines) (filter (lambda (lexeme) (> (string-length lexeme) 1)) (append-map lexemes-from-line lines))) +(def (average-word-length words) + (if (null? words) + 0.0 + (/ (exact->inexact (sum (map string-length words))) (length words)))) +(def (sentence-punctuation-count text) + (for/fold ([n 0]) ([ch (in-string text)]) + (if (or (char=? ch #\.) (char=? ch #\!) (char=? ch #\?)) + (+ n 1) + n))) +(def (message-naturalness-signal subject body) + (let* ([text (down (string-join (list subject body) "\n"))] + [words (source-lexemes (list subject body))] + [unique-words (unique words)] + [word-total (length words)] + [unique-ratio (if (= word-total 0) 1.0 (/ (exact->inexact (length unique-words)) word-total))] + [avg-word-length (average-word-length words)] + [sentence-count (sentence-punctuation-count text)] + [phrase-count (length (filter (lambda (p) (contains? text p)) ai-message-phrases))] + [noise? (any-contains? text human-noise)] + [low-diversity? (and (>= word-total 24) (< unique-ratio 0.55))] + [formulaic? (>= phrase-count 2)] + [over-polished? (and (>= word-total 24) (>= sentence-count 3) (> avg-word-length 5.0) (not noise?))] + [score (+ (if low-diversity? 0.10 0.0) + (if formulaic? 0.12 0.0) + (if over-polished? 0.08 0.0))] + [evidence (append (if low-diversity? + (list (str "offline message naturalness low unique-word ratio " unique-ratio)) + '()) + (if formulaic? + (list (str "formulaic generated-message phrase count " phrase-count)) + '()) + (if over-polished? + (list (str "polished long message proxy avg-word-length " avg-word-length)) + '()))]) + (sig "message-naturalness" "text" score 0.08 "low" + "offline message naturalness proxy found formulaic or unusually smooth text" + evidence + "weak offline approximation; not GPT-2 perplexity and not proof of AI authorship"))) (def semantic-stopwords '("the" "and" "for" "with" "that" "this" "from" "into" "onto" "are" "was" "were" "has" "have" "had" "will" "can" "could" "should" "would" "a" "an" "to" "of" "in" "on" "by" "as" "is" "it" "be" "or" "if" "else" "return" "def" "class" "let" "const" "var" "function" "value" "values" "data" "result" "item" "items" "temp")) (def (semantic-term? term) @@ -986,7 +1024,7 @@ [sim-pair (similarity-signal lines hashes author-email)] [raw-signals (if eligible? (append - (list (message-signal subject body adds) (code-signal lines) (cadence-code-pattern-signal lines) + (list (message-signal subject body adds) (message-naturalness-signal subject body) (code-signal lines) (cadence-code-pattern-signal lines) (structure-signal paths adds dels lines) (cadence-diff-shape-signal files paths adds dels) (semantic-alignment-signal subject body lines) @@ -1161,6 +1199,7 @@ (cons 'limitations limitations))) (def strategy-catalog (list (strategy-catalog-entry "message-style" "text" 0.10 "weight_text" "message style alone is weak evidence") + (strategy-catalog-entry "message-naturalness" "text" 0.08 "weight_text" "weak offline approximation; not GPT-2 perplexity") (strategy-catalog-entry "lexical-code-style" "code" 0.35 "weight_code" "language-neutral lexical analysis is noisy") (strategy-catalog-entry "cadence-code-patterns" "code" 0.18 "weight_code" "framework conventions and mechanical scaffolds can match") (strategy-catalog-entry "python-ast-style" "code" 0.20 "weight_code" "dependency-free Python analysis approximates AST features") --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -40,7 +40,8 @@ git -C "$fixture" add src/generated.py git -C "$fixture" add vendor/library.py GIT_AUTHOR_DATE='2026-07-29T09:01:00-06:00' \ GIT_COMMITTER_DATE='2026-07-29T09:01:00-06:00' \ - git -C "$fixture" commit -q -m 'feat: Implement robust generated helpers.' + git -C "$fixture" commit -q -m 'feat: Implement robust generated helpers.' \ + -m 'This commit introduces a comprehensive helper implementation designed to ensure that users can process values. It provides a robust helper workflow designed to ensure that users can process values. It allows users to process values with comprehensive helper behavior.' git -C "$fixture" notes --ref=ai add -m '{"tool":"codex","model":"gpt-5","lines":[{"path":"src/generated.py","start":1,"end":360}]}' HEAD json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 2) @@ -57,6 +58,9 @@ printf '%s\n' "$json" | grep -q '"file_findings":\[{"path":"src/generated.py","r printf '%s\n' "$json" | grep -q '"attribution_status":"matched-note-range"' printf '%s\n' "$json" | grep -q '"unmatched_recorded_attribution":\[\]' printf '%s\n' "$json" | grep -q '"metadata_hits":\["codex","openai"\]' +printf '%s\n' "$json" | grep -q '"name":"message-naturalness"' +printf '%s\n' "$json" | grep -q 'formulaic generated-message phrase count' +printf '%s\n' "$json" | grep -q 'weak offline approximation; not GPT-2 perplexity' printf '%s\n' "$json" | grep -q '"category":"code"' printf '%s\n' "$json" | grep -q '"name":"cadence-code-patterns"' printf '%s\n' "$json" | grep -q 'regular generated-style names' @@ -275,6 +279,18 @@ printf '%s\n' "$local_llm_json" | grep -q '"name":"local-llm-opinion"' printf '%s\n' "$local_llm_json" | grep -q '"category":"model"' printf '%s\n' "$local_llm_json" | grep -q '"evidence":\["provider saw generated.py"\]' +malformed_provider_script="$provider_tmp/malformed-provider.sh" +{ + printf '#!/usr/bin/env sh\n' + printf 'printf '\''not-json\\n'\''\n' +} > "$malformed_provider_script" +chmod +x "$malformed_provider_script" +malformed_provider_config="$fixture/malformed-local-provider.json" +printf '{"local_provider_command":["%s"]}\n' "$malformed_provider_script" > "$malformed_provider_config" +malformed_provider_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$malformed_provider_config" --format json --count 1 --llm --provider local) +printf '%s\n' "$malformed_provider_json" | grep -q '"llm_used":false' +printf '%s\n' "$malformed_provider_json" | grep -q 'local provider returned malformed JSON or omitted numeric score' + no_llm_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --llm --provider local --no-llm) if printf '%s\n' "$no_llm_json" | grep -q 'local provider requested'; then echo "no-llm should disable requested LLM warning" >&2