Honor generated path exclusions
ober
60a73f3a005b3dc2c4e19628096840ecb651dadf
--- a/README.md +++ b/README.md @@ -31,17 +31,21 @@ jerboa main-binary.ss scan /path/to/repo --format json ``` Supported options are `--config FILE`, `--count N`, `--all`, `--from REV`, -`--to REV`, `--commit REV`, `--file PATH`, `--min-lines N`, -`--format table|json|jsonl|markdown`, `--metadata-only`, and -`--heuristics-only`. +`--to REV`, `--commit REV`, `--file PATH`, `--include PATH`, +`--exclude PREFIX`, `--min-lines N`, `--format table|json|jsonl|markdown`, +`--metadata-only`, and `--heuristics-only`. `--config FILE` reads guarded JSON. Supported scan keys are `path`, `count`, -`format`, `from`, `to`, `file`, `min_lines`, `metadata_only`, and +`format`, `from`, `to`, `file`, `exclude`, `min_lines`, `metadata_only`, and `heuristics_only`. Supported scoring keys are `human_threshold`, `ai_threshold`, `weight_text`, `weight_code`, `weight_structure`, `weight_similarity`, `weight_history`, and `weight_baseline`. Later CLI flags override earlier config values. +By default, scans skip `vendor/`, `generated/`, `dist/`, `node_modules/`, and +`.git/` paths. Use `--include PATH` or `--file PATH` to inspect one of those +paths explicitly. + ## What It Reads The scanner runs Git commands against the requested repository using fixed --- a/main-binary.ss +++ b/main-binary.ss @@ -23,7 +23,7 @@ xs))) (def (usage) - (displayln "usage: jerboa main-binary.ss scan [PATH] [--config FILE] [--count N|--all] [--from REV] [--to REV] [--file PATH] [--min-lines N] [--format table|json|jsonl|markdown]") + (displayln "usage: jerboa main-binary.ss scan [PATH] [--config FILE] [--count N|--all] [--from REV] [--to REV] [--file PATH|--include PATH] [--exclude PREFIX] [--min-lines N] [--format table|json|jsonl|markdown]") (displayln " jerboa main-binary.ss explain REV [PATH] [--format json|markdown|table]") (displayln " jerboa main-binary.ss stats [PATH] [--count N]") (displayln " jerboa main-binary.ss verify-authorship [PATH] [--count N]")) @@ -69,6 +69,23 @@ (def (maybe-append xs ys) (if (null? ys) xs (append xs ys))) +(def default-excludes '("vendor/" "generated/" "dist/" "node_modules/" ".git/")) +(def current-excludes default-excludes) + +(def (public-prefix? prefix text) + (and (<= (string-length prefix) (string-length text)) + (let loop ([i 0]) + (or (= i (string-length prefix)) + (and (= (char->integer (string-ref prefix i)) (char->integer (string-ref text i))) + (loop (+ i 1))))))) + +(def (excluded-path? path) + (for/or ([prefix current-excludes]) (public-prefix? prefix path))) + +(def (selected-file-records records explicit-file) + (if explicit-file + records + (filter (lambda (record) (not (excluded-path? (car record)))) records))) (def (pathspec-args file) (if file (list "--" file) '())) @@ -152,11 +169,12 @@ (list path adds dels))) (def (changed-files repo rev file) - (let ([args (append (list "show" "--format=" "--numstat" "--first-parent" rev) - (pathspec-args file))]) - (map parse-numstat - (filter (lambda (line) (not (blank? line))) - (split-lines (git repo args)))))) + (let* ([args (append (list "show" "--format=" "--numstat" "--first-parent" rev) + (pathspec-args file))] + [records (map parse-numstat + (filter (lambda (line) (not (blank? line))) + (split-lines (git repo args))))]) + (selected-file-records records file))) (def (numstat-adds files) (sum (map cadr files))) (def (numstat-dels files) (sum (map caddr files))) @@ -169,6 +187,10 @@ (filter (lambda (line) (and (string-prefix? "+" line) (not (string-prefix? "+++" line)))) (split-lines (git repo args)))))) +(def (added-lines-for-paths repo rev paths explicit-file) + (if explicit-file + (added-lines repo rev explicit-file) + (append-map (lambda (path) (added-lines repo rev path)) paths))) (def known-agents '("codex" "copilot" "claude" "cursor" "openai" "anthropic" "aider" "windsurf" "cody" "tabnine" "ai-agent")) @@ -422,7 +444,7 @@ [author-name (safe-ref fields 2 "")] [author-email (safe-ref fields 3 "")] [time (parse-int (safe-ref fields 4 "0") 0)] [subject (safe-ref fields 5 "")] [body (commit-message repo rev)] [files (changed-files repo rev file)] [paths (numstat-paths files)] - [adds (numstat-adds files)] [dels (numstat-dels files)] [lines (added-lines repo rev file)] + [adds (numstat-adds files)] [dels (numstat-dels files)] [lines (added-lines-for-paths repo rev paths file)] [note (note-text repo rev)] [note-obj (parse-note-object note)] [attribution (note-attributions note-obj)] [metadata (metadata-hits author-name author-email subject body note)] @@ -549,6 +571,9 @@ (def (config-bool obj field fallback) (let ([v (config-value obj field fallback)]) (if (boolean? v) v fallback))) +(def (config-string-list obj field fallback) + (let ([v (config-value obj field fallback)]) + (if (list? v) (filter string? v) fallback))) (def (read-config-object path) (let ([content (try-result (read-file-string path))]) @@ -570,6 +595,7 @@ (if (hash-table? obj) (begin (set! current-config (config-scan-config obj)) + (set! current-excludes (append (config-string-list obj "exclude" '()) current-excludes)) (make-options (options-command opts) (config-string obj "path" (options-path opts)) (config-number obj "count" (options-count opts)) @@ -630,12 +656,16 @@ (options-commit opts) (options-from opts) (cadr xs) (options-file opts) (options-min-lines opts) (options-metadata-only? opts) (options-heuristics-only? opts)) path-set?)] - [(and (string=? (car xs) "--file") (pair? (cdr xs))) + [(and (or (string=? (car xs) "--file") (string=? (car xs) "--include")) (pair? (cdr xs))) (loop (cddr xs) (make-options (options-command opts) (options-path opts) (options-count opts) (options-format opts) (options-commit opts) (options-from opts) (options-to opts) (cadr xs) (options-min-lines opts) (options-metadata-only? opts) (options-heuristics-only? opts)) path-set?)] + [(and (string=? (car xs) "--exclude") (pair? (cdr xs))) + (begin + (set! current-excludes (cons (cadr xs) current-excludes)) + (loop (cddr xs) opts path-set?))] [(and (string=? (car xs) "--min-lines") (pair? (cdr xs))) (loop (cddr xs) (make-options (options-command opts) (options-path opts) (options-count opts) (options-format opts) --- a/tests/fixture-smoke.sh +++ b/tests/fixture-smoke.sh @@ -18,6 +18,7 @@ GIT_COMMITTER_DATE='2026-07-29T09:00:00-06:00' \ git -C "$fixture" config user.name "Codex" git -C "$fixture" config user.email "codex@openai.example" mkdir -p "$fixture/src" +mkdir -p "$fixture/vendor" i=1 while [ "$i" -le 90 ]; do { @@ -28,7 +29,9 @@ while [ "$i" -le 90 ]; do } >> "$fixture/src/generated.py" i=$((i + 1)) done +printf 'def vendored_helper(value):\n return value\n' > "$fixture/vendor/library.py" git -C "$fixture" add src/generated.py +git -C "$fixture" add vendor/library.py GIT_AUTHOR_DATE='2026-07-29T09:01:00-06:00' \ GIT_COMMITTER_DATE='2026-07-29T09:01:00-06:00' \ git -C "$fixture" commit -q -m 'feat: Implement robust generated helpers.' @@ -48,6 +51,10 @@ printf '%s\n' "$json" | grep -q '"category":"history"' printf '%s\n' "$json" | grep -q '"category":"baseline"' printf '%s\n' "$json" | grep -q '"evidence":\[\]' printf '%s\n' "$json" | grep -q '"warnings":\[\]' +if printf '%s\n' "$json" | grep -q 'vendor/library.py'; then + echo "default scan should exclude vendored paths" >&2 + exit 1 +fi heuristics=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --heuristics-only) printf '%s\n' "$heuristics" | grep -q '"recorded_ai_note_present":true' @@ -81,6 +88,14 @@ file_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 5 --f printf '%s\n' "$file_json" | grep -q '"count":1' printf '%s\n' "$file_json" | grep -q '"files":\["src/generated.py"\]' +include_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 5 --include vendor/library.py) +printf '%s\n' "$include_json" | grep -q '"count":1' +printf '%s\n' "$include_json" | grep -q '"files":\["vendor/library.py"\]' + +exclude_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --exclude src/) +printf '%s\n' "$exclude_json" | grep -q '"files":\[\]' +printf '%s\n' "$exclude_json" | grep -q 'no changed text files found or commit is unavailable' + min_json=$("$root/bin/jerboa-aigit" scan "$fixture" --format json --count 1 --min-lines 999) printf '%s\n' "$min_json" | grep -q '"signals":\[\]' printf '%s\n' "$min_json" | grep -q 'heuristics skipped below --min-lines 999' @@ -95,6 +110,10 @@ config_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file") printf '%s\n' "$config_json" | grep -q '"count":1' printf '%s\n' "$config_json" | grep -q '"files":\["src/generated.py"\]' +printf '{"format":"json","count":1,"exclude":["src/"]}\n' > "$config_file" +config_exclude_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file") +printf '%s\n' "$config_exclude_json" | grep -q '"files":\[\]' + printf '{"format":"json","count":1,"weight_code":0.0,"weight_text":0.0,"weight_structure":0.0,"weight_similarity":0.0,"weight_history":0.0,"ai_threshold":0.9,"human_threshold":0.1}\n' > "$config_file" weighted_json=$("$root/bin/jerboa-aigit" scan "$fixture" --config "$config_file" --heuristics-only) printf '%s\n' "$weighted_json" | grep -q '"config_hash":"[0-9]'