Add Python-specific code heuristics
ober
f94ba85284e4f45a9aeeaf0d2a0f73489a69b67b
--- a/GAPS.md +++ b/GAPS.md @@ -132,6 +132,11 @@ Acceptance criteria: `sniff` has Python AST docstring/function analysis. This scanner only performs language-neutral lexical checks. +Status: implemented as dependency-free Python AST-style analysis over added +`.py`/`.pyi` lines only, with evidence for generated docstrings, repetitive +function structure, regular function-length gaps, low identifier diversity, and +non-Python fallback coverage. + Acceptance criteria: - Add Python-specific heuristics only for Python changed files. --- a/main-binary.ss +++ b/main-binary.ss @@ -459,6 +459,77 @@ phrases)]) (sig "lexical-code-style" "code" score 0.35 "medium" "added code has lexical/comment patterns associated with generated scaffolding" evidence "language-neutral lexical analysis is noisy"))) +(def python-path-suffixes '(".py" ".pyi")) +(def (python-path? path) + (suffix-any? (down path) python-path-suffixes)) +(def python-keywords + '("and" "as" "assert" "async" "await" "break" "class" "continue" "def" "del" "elif" "else" "except" "false" "finally" "for" "from" "global" "if" "import" "in" "is" "lambda" "none" "nonlocal" "not" "or" "pass" "raise" "return" "true" "try" "while" "with" "yield" "self" "cls")) +(def (python-def-line? line) + (let ([trimmed (string-trim line)]) + (or (string-prefix? "def " trimmed) + (string-prefix? "async def " trimmed)))) +(def (python-return-line? line) + (string-prefix? "return " (string-trim line))) +(def (python-docstring-line? line) + (let ([trimmed (string-trim line)]) + (or (contains? trimmed "\"\"\"") + (contains? trimmed "'''")))) +(def (line-indexes-where pred lines) + (let loop ([xs lines] [idx 0] [out '()]) + (cond [(null? xs) (reverse out)] + [(pred (car xs)) (loop (cdr xs) (+ idx 1) (cons idx out))] + [else (loop (cdr xs) (+ idx 1) out)]))) +(def (adjacent-differences xs) + (let loop ([ys xs] [out '()]) + (if (or (null? ys) (null? (cdr ys))) + (reverse out) + (loop (cdr ys) (cons (- (cadr ys) (car ys)) out))))) +(def (python-identifier-lexemes lines) + (filter (lambda (lexeme) (not (member lexeme python-keywords))) + (source-lexemes lines))) +(def (python-lines-for-commit repo rev paths explicit-file) + (cond [(and explicit-file (python-path? explicit-file)) + (bounded-added-lines-for-paths repo rev paths explicit-file)] + [explicit-file '()] + [else (bounded-added-lines-for-paths repo rev (filter python-path? paths) #f)])) +(def (python-signals lines) + (let* ([nonblank (filter (lambda (line) (not (blank? line))) lines)] + [text (down (string-join nonblank "\n"))] + [def-indexes (line-indexes-where python-def-line? nonblank)] + [function-gaps (adjacent-differences def-indexes)] + [def-count (length def-indexes)] + [return-count (count-where python-return-line? nonblank)] + [docstring-count (count-where python-docstring-line? nonblank)] + [doc-phrases (filter (lambda (p) (contains? text p)) explanatory-phrases)] + [identifiers (python-identifier-lexemes nonblank)] + [unique-identifiers (unique identifiers)] + [diversity (if (null? identifiers) 1.0 (/ (exact->inexact (length unique-identifiers)) (length identifiers)))] + [generated-docstrings? (and (>= docstring-count 2) (pair? doc-phrases))] + [repetitive-functions? (and (>= def-count 5) (>= return-count def-count))] + [regular-lengths? (and (>= (length function-gaps) 4) (< (stddev function-gaps) 1.50))] + [low-diversity? (and (>= (length identifiers) 60) (< diversity 0.22))] + [score (+ (if generated-docstrings? 0.22 0.0) + (if repetitive-functions? 0.20 0.0) + (if regular-lengths? 0.18 0.0) + (if low-diversity? 0.20 0.0))] + [evidence (append (if generated-docstrings? + (list (str "generated-style Python docstrings " docstring-count)) + '()) + (if repetitive-functions? + (list (str "repetitive Python function structure " def-count " defs / " return-count " returns")) + '()) + (if regular-lengths? + (list (str "regular Python function length gaps stddev " (stddev function-gaps))) + '()) + (if low-diversity? + (list (str "low Python identifier diversity " diversity)) + '()))]) + (if (null? nonblank) + '() + (list (sig "python-ast-style" "code" score 0.20 "medium" + "Python-specific docstring, function-structure, and identifier patterns matched" + evidence + "dependency-free textual Python analysis approximates AST features and can match generated or repetitive human code"))))) (def error-handling-terms '("try" "catch" "except" "raise" "throw" "throws" "error" "errors" "exception" "fallback" "retry")) (def scaffold-name-fragments '("helper" "handler" "manager" "processor" "service" "client" "controller" "factory" "adapter")) (def (contains-any-fragment? text fragments) @@ -907,19 +978,22 @@ [adds (numstat-adds files)] [dels (numstat-dels files)] [line-count (added-line-count-for-paths repo rev paths file)] [lines (bounded-added-lines-for-paths repo rev paths file)] + [python-lines (python-lines-for-commit repo rev paths file)] [note-count (note-byte-count repo rev)] [note (bounded-note-text repo rev)] [note-obj (parse-note-object note)] [attribution (note-attributions note-obj)] [metadata (metadata-hits author-name author-email subject body note)] [eligible? (and (pair? files) (pair? lines) (or (= min-lines 0) (>= (length lines) min-lines)))] [sim-pair (similarity-signal lines hashes author-email)] [raw-signals (if eligible? - (list (message-signal subject body adds) (code-signal lines) (cadence-code-pattern-signal lines) - (structure-signal paths adds dels lines) - (cadence-diff-shape-signal files paths adds dels) - (semantic-alignment-signal subject body lines) - (car sim-pair) (history-signal adds time (parent-time repo parent) (author-events repo revs author-email file)) - (baseline-signal adds (prior-additions repo revs author-email rev file)) - (repository-baseline-signal adds (prior-repository-additions repo revs rev file))) + (append + (list (message-signal subject body adds) (code-signal lines) (cadence-code-pattern-signal lines) + (structure-signal paths adds dels lines) + (cadence-diff-shape-signal files paths adds dels) + (semantic-alignment-signal subject body lines) + (car sim-pair) (history-signal adds time (parent-time repo parent) (author-events repo revs author-email file)) + (baseline-signal adds (prior-additions repo revs author-email rev file)) + (repository-baseline-signal adds (prior-repository-additions repo revs rev file))) + (python-signals python-lines)) '())] [weighted-signals (if metadata-only? '() (map (lambda (s) (with-config-weight current-config s)) raw-signals))] [preliminary-score (if metadata-only? 0.0 (aggregate-score weighted-signals))] @@ -1089,6 +1163,7 @@ (list (strategy-catalog-entry "message-style" "text" 0.10 "weight_text" "message style alone is weak evidence") (strategy-catalog-entry "lexical-code-style" "code" 0.35 "weight_code" "language-neutral lexical analysis is noisy") (strategy-catalog-entry "cadence-code-patterns" "code" 0.18 "weight_code" "framework conventions and mechanical scaffolds can match") + (strategy-catalog-entry "python-ast-style" "code" 0.20 "weight_code" "dependency-free Python analysis approximates AST features") (strategy-catalog-entry "diff-structure" "structure" 0.15 "weight_structure" "formatters, generated files, and refactors can look similar") (strategy-catalog-entry "cadence-diff-shape" "structure" 0.15 "weight_structure" "large generated files, vendored updates, and mechanical refactors can match") (strategy-catalog-entry "semantic-alignment" "semantic" 0.10 "weight_semantic" "offline lexical overlap is a weak proxy for semantic self-summary") --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -60,6 +60,11 @@ printf '%s\n' "$json" | grep -q '"metadata_hits":\["codex","openai"\]' printf '%s\n' "$json" | grep -q '"category":"code"' printf '%s\n' "$json" | grep -q '"name":"cadence-code-patterns"' printf '%s\n' "$json" | grep -q 'regular generated-style names' +printf '%s\n' "$json" | grep -q '"name":"python-ast-style"' +printf '%s\n' "$json" | grep -q 'generated-style Python docstrings' +printf '%s\n' "$json" | grep -q 'repetitive Python function structure' +printf '%s\n' "$json" | grep -q 'regular Python function length gaps' +printf '%s\n' "$json" | grep -q 'low Python identifier diversity' printf '%s\n' "$json" | grep -q '"category":"structure"' printf '%s\n' "$json" | grep -q '"name":"cadence-diff-shape"' printf '%s\n' "$json" | grep -q '"name":"semantic-alignment"' @@ -102,6 +107,10 @@ shape_json=$("$root/bin/jerboa-aigit" scan "$shape_fixture" --format json --coun printf '%s\n' "$shape_json" | grep -q '"name":"cadence-diff-shape"' printf '%s\n' "$shape_json" | grep -q 'skewed addition/deletion ratio' printf '%s\n' "$shape_json" | grep -q 'template/generated path count' +if printf '%s\n' "$shape_json" | grep -q '"name":"python-ast-style"'; then + echo "non-Python fixture should not emit Python-specific signal" >&2 + exit 1 +fi git -C "$timing_fixture" init -q git -C "$timing_fixture" config user.name "Burst Bot" @@ -382,10 +391,10 @@ weighted_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" printf '%s\n' "$weighted_json" | grep -q '"config_hash":"[0-9]' printf '%s\n' "$weighted_json" | grep -q '"category":"code","score":[0-9.]*,"weight":0.0' -threshold_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --heuristics-only --threshold 0.99,1.0) +threshold_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" --format json --count 1 --heuristics-only --threshold 0.99,1.0) printf '%s\n' "$threshold_json" | grep -q '"verdict":"likely-human-style"' -threshold_triplet_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --heuristics-only --threshold 0.99,0.995,1.0) +threshold_triplet_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" --format json --count 1 --heuristics-only --threshold 0.99,0.995,1.0) printf '%s\n' "$threshold_triplet_json" | grep -q '"verdict":"likely-human-style"' git -C "$fixture" notes --ref=ai remove HEAD >/dev/null