Extract Semgrep source offset utilities
ober
f0efb009fa96327c66f99edbc0ffe457aee6ad0a
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -455,6 +455,8 @@ Completed in the repo: - extracted path/file filtering into `src/semgrep/targeting/path-filter.ss` - extracted rule applicability and parse gating into `src/semgrep/engine/rule-plan.ss` + - extracted source offset and line/column conversions into + `src/semgrep/source/offsets.ss` Validation at this checkpoint: --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -17,7 +17,8 @@ (tree-sitter tree-sitter) (semgrep lang) (semgrep rule) (semgrep result) (semgrep engine rule-plan) (semgrep rule parse-rule) (semgrep parse parse-target) - (semgrep targeting path-filter) (semgrep match structural)) + (semgrep source offsets) (semgrep targeting path-filter) + (semgrep match structural)) (def (alist-ref/default xs key default) (let ([found (assoc key xs)]) (if found (cdr found) default))) @@ -44,52 +45,6 @@ [(null? xs) '()] [(eq? target (car xs)) (cdr xs)] [else (cons (car xs) (remove-first-eq target (cdr xs)))])) - (def (offset->line-col source offset) - (let ([len (string-length source)]) - (let loop ([i 0] [line 1] [col 1]) - (cond - [(or (= i offset) (= i len)) (values line col)] - [(char=? (string-ref source i) #\newline) - (loop (+ i 1) (+ line 1) 1)] - [else - (loop - (+ i 1) - line - (+ col - (semgrep-source-byte-length - (string-ref source i))))])))) - (def (char-utf8-byte-length ch) - (let ([codepoint (char->integer ch)]) - (cond - [(<= codepoint 127) 1] - [(<= codepoint 2047) 2] - [(<= codepoint 65535) 3] - [else 4]))) - (def (semgrep-source-byte-length ch) - (if (char=? ch #\�) 1 (char-utf8-byte-length ch))) - (def (source-index->semgrep-offset source index) - (let ([len (string-length source)]) - (let loop ([i 0] [offset 0]) - (cond - [(or (>= i index) (>= i len)) offset] - [else - (loop - (+ i 1) - (+ offset - (semgrep-source-byte-length - (string-ref source i))))])))) - (def (tree-byte-offset->source-index source offset) - (let ([len (string-length source)]) - (let loop ([i 0] [tree-offset 0]) - (cond - [(or (>= i len) (>= tree-offset offset)) i] - [else - (let* ([next-offset (+ tree-offset - (char-utf8-byte-length - (string-ref source i)))]) - (if (> next-offset offset) - i - (loop (+ i 1) next-offset)))])))) (def (regex-capture-name-char? ch) (or (char-alphabetic? ch) (char-numeric? ch) new file mode 100644 --- /dev/null +++ b/lib/semgrep/source/offsets.sls @@ -0,0 +1,60 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/source/offsets.ss + +(library (semgrep source offsets) + (export offset->line-col char-utf8-byte-length + semgrep-source-byte-length source-index->semgrep-offset + tree-byte-offset->source-index) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?)) + (def (offset->line-col source offset) + (let ([len (string-length source)]) + (let loop ([i 0] [line 1] [col 1]) + (cond + [(or (= i offset) (= i len)) (values line col)] + [(char=? (string-ref source i) #\newline) + (loop (+ i 1) (+ line 1) 1)] + [else + (loop + (+ i 1) + line + (+ col + (semgrep-source-byte-length + (string-ref source i))))])))) + (def (char-utf8-byte-length ch) + (let ([codepoint (char->integer ch)]) + (cond + [(<= codepoint 127) 1] + [(<= codepoint 2047) 2] + [(<= codepoint 65535) 3] + [else 4]))) + (def (semgrep-source-byte-length ch) + (if (char=? ch #\�) 1 (char-utf8-byte-length ch))) + (def (source-index->semgrep-offset source index) + (let ([len (string-length source)]) + (let loop ([i 0] [offset 0]) + (cond + [(or (>= i index) (>= i len)) offset] + [else + (loop + (+ i 1) + (+ offset + (semgrep-source-byte-length + (string-ref source i))))])))) + (def (tree-byte-offset->source-index source offset) + (let ([len (string-length source)]) + (let loop ([i 0] [tree-offset 0]) + (cond + [(or (>= i len) (>= tree-offset offset)) i] + [else + (let* ([next-offset (+ tree-offset + (char-utf8-byte-length + (string-ref source i)))]) + (if (> next-offset offset) + i + (loop (+ i 1) next-offset)))]))))) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -1,17 +1,16 @@ -(("src/semgrep/targeting/path-filter.ss" - . - "9900721941C6B96") ("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") +(("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") ("src/semgrep/result.ss" . "22D23E40B49BA529") + ("src/semgrep/engine/rule-plan.ss" . "6631789392AB5F80") + ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") + ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") + ("src/semgrep/targeting/path-filter.ss" . "9900721941C6B96") ("src/semgrep/rule/parse-rule.ss" . "EFA6D401699CEDEF") - ("src/semgrep/result.ss" . "22D23E40B49BA529") ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") - ("src/semgrep/engine/rule-plan.ss" . "6631789392AB5F80") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/scan.ss" . "E2D828980E2073F1") + ("src/semgrep/scan.ss" . "70A082557A5C7744") ("src/semgrep/rule.ss" . "E12C108153C181FA") ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") - ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") - ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") + ("src/semgrep/source/offsets.ss" . "834EFDB706823794") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -13,6 +13,7 @@ (semgrep engine rule-plan) (semgrep rule parse-rule) (semgrep parse parse-target) + (semgrep source offsets) (semgrep targeting path-filter) (semgrep match structural)) @@ -50,56 +51,6 @@ [(eq? target (car xs)) (cdr xs)] [else (cons (car xs) (remove-first-eq target (cdr xs)))])) -(def (offset->line-col source offset) - (let ([len (string-length source)]) - (let loop ([i 0] [line 1] [col 1]) - (cond - [(or (= i offset) (= i len)) (values line col)] - [(char=? (string-ref source i) #\newline) - (loop (+ i 1) (+ line 1) 1)] - [else - (loop (+ i 1) - line - (+ col - (semgrep-source-byte-length - (string-ref source i))))])))) - -(def (char-utf8-byte-length ch) - (let ([codepoint (char->integer ch)]) - (cond - [(<= codepoint #x7f) 1] - [(<= codepoint #x7ff) 2] - [(<= codepoint #xffff) 3] - [else 4]))) - -(def (semgrep-source-byte-length ch) - (if (char=? ch #\xfffd) - 1 - (char-utf8-byte-length ch))) - -(def (source-index->semgrep-offset source index) - (let ([len (string-length source)]) - (let loop ([i 0] [offset 0]) - (cond - [(or (>= i index) (>= i len)) offset] - [else - (loop (+ i 1) - (+ offset - (semgrep-source-byte-length (string-ref source i))))])))) - -(def (tree-byte-offset->source-index source offset) - (let ([len (string-length source)]) - (let loop ([i 0] [tree-offset 0]) - (cond - [(or (>= i len) (>= tree-offset offset)) i] - [else - (let* ([next-offset - (+ tree-offset - (char-utf8-byte-length (string-ref source i)))]) - (if (> next-offset offset) - i - (loop (+ i 1) next-offset)))])))) - (def (regex-capture-name-char? ch) (or (char-alphabetic? ch) (char-numeric? ch) new file mode 100644 --- /dev/null +++ b/src/semgrep/source/offsets.ss @@ -0,0 +1,58 @@ +(export + offset->line-col + char-utf8-byte-length + semgrep-source-byte-length + source-index->semgrep-offset + tree-byte-offset->source-index) + +(import (except (jerboa prelude) meta atom?)) + +(def (offset->line-col source offset) + (let ([len (string-length source)]) + (let loop ([i 0] [line 1] [col 1]) + (cond + [(or (= i offset) (= i len)) (values line col)] + [(char=? (string-ref source i) #\newline) + (loop (+ i 1) (+ line 1) 1)] + [else + (loop (+ i 1) + line + (+ col + (semgrep-source-byte-length + (string-ref source i))))])))) + +(def (char-utf8-byte-length ch) + (let ([codepoint (char->integer ch)]) + (cond + [(<= codepoint #x7f) 1] + [(<= codepoint #x7ff) 2] + [(<= codepoint #xffff) 3] + [else 4]))) + +(def (semgrep-source-byte-length ch) + (if (char=? ch #\xfffd) + 1 + (char-utf8-byte-length ch))) + +(def (source-index->semgrep-offset source index) + (let ([len (string-length source)]) + (let loop ([i 0] [offset 0]) + (cond + [(or (>= i index) (>= i len)) offset] + [else + (loop (+ i 1) + (+ offset + (semgrep-source-byte-length (string-ref source i))))])))) + +(def (tree-byte-offset->source-index source offset) + (let ([len (string-length source)]) + (let loop ([i 0] [tree-offset 0]) + (cond + [(or (>= i len) (>= tree-offset offset)) i] + [else + (let* ([next-offset + (+ tree-offset + (char-utf8-byte-length (string-ref source i)))]) + (if (> next-offset offset) + i + (loop (+ i 1) next-offset)))]))))