data: repair catalog structure + add P2 perf artifacts
ober
ec474be7fce57ecdf28d6519102b3d4bfc266f70
--- a/data/anti-patterns.sexp +++ b/data/anti-patterns.sexp @@ -4962,4 +4962,115 @@ ("title" . "Positional records in code generators (off-by-one magnet)") - ("tools" "jerboa_howto"))) + ("tools" "jerboa_howto")) + (("advice" + . + "Check the app spec for (asset-dir \"path\") entries. Ensure the referenced directory exists relative to the spec file and contains all files that the code reads via assetManager.open(). Copy assets from the original app if migrating to typed. ALWAYS test on emulator - asset errors are runtime-only.") + ("avoid" + . + "Do not assume assets are automatically included. The typed generator only copies assets from directories listed in (asset-dir) entries in the app spec. Missing assets cause FileNotFoundException at runtime, not compile time.") + ("id" . "typed-android-missing-asset-dir") + ("kinds" "android" "typed" "build") + ("pattern" + "FileNotFoundException.*\\.csv|AssetManager.*open|box_types") + ("severity" "critical") + ("tags" "android" "assets" "crash" "runtime" "typed" + "generator") + ("title" + . + "Missing asset-dir in typed Android app spec causes runtime crash") + ("tools" "jerboa_verify" "jerboa_compile_check")) + (("advice" + . + "Create the Intent with (intentForAction ...), then use a begin block to call intentTypeSetRaw, intentDataSetRaw, intentAddFlagsRaw in sequence. Finally call mainActivityStartActivityRaw with the intent. See cookbook recipe typed-android-intent-side-effects.") + ("avoid" + . + "Do not nest intentTypeSetRaw, intentDataSetRaw, or intentAddFlagsRaw as function arguments. They return Unit, not Intent. Chaining them causes argument-type-mismatch errors in the typed checker.") + ("id" . "typed-android-intent-chaining") + ("kinds" "android" "typed" "code") + ("pattern" + "intentAddFlagsRaw.*intentDataSetRaw|intentDataSetRaw.*intentTypeSetRaw") + ("severity" "high") + ("tags" "android" "intent" "typed" "unit" "side-effect" + "begin") + ("title" + . + "Chaining typed Android Intent functions causes type errors") + ("tools" "jerboa_verify" "jerboa_function_signature")) + (("advice" + . + "After make ssd-compile and assembleDebug, run: adb install -r app-debug.apk && adb shell am start -n com.sfb.ssdreview/.MainActivity && sleep 4 && adb logcat -d -s AndroidRuntime:E. The output MUST be empty. Also verify with: adb shell dumpsys activity activities | grep topResumedActivity. See AGENTS.md in jerboa-android-typed for the full protocol.") + ("avoid" + . + "Do not ship APKs to the user without emulator testing. Do not scp to termux as the only verification step. The human should never discover a crash by installing the APK themselves.") + ("id" . "typed-android-skip-emulator-test") + ("kinds" "android" "typed" "workflow") + ("pattern" "scp.*termux.*apk|push.*apk.*device") + ("severity" "critical") + ("tags" "android" "emulator" "adb" "test" "workflow" + "mandatory") + ("title" + . + "Shipping typed Android APK without emulator testing") + ("tools" "jerboa_verify")) + (("advice" + . + "Add the function name to the (export ...) list in the typed-library form. The export list is at the top of the typed-kotlin-file fragment. Functions called from other modules MUST be exported.") + ("avoid" + . + "Do not define functions in typed-library forms without adding them to the (export ...) list. Unexported functions cause cryptic unknown-value errors at compile time, not at the definition site but at the call site in other modules.") + ("id" . "typed-android-missing-export") + ("kinds" "android" "typed" "code") + ("pattern" "unknown-value.*typed modules have check errors") + ("severity" "high") + ("tags" "android" "typed" "export" "unknown-value" + "compile") + ("title" + . + "Missing export in typed-library causes unknown-value errors") + ("tools" "jerboa_verify" "jerboa_compile_check")) + (("advice" + . + "Allocate ONE buffer outside the loop and reuse/refill it (a per-fd buffer hashtable, dropped on close). Preallocate the exact-size output once (count first, then make-bytevector/make-string + fill). For per-position substring in searches, use a non-allocating char-by-char compare or string-contains (no per-position substring). Use bytevector-copy!/bytevector-fill! on a reused buffer instead of fresh bytevectors.") + ("avoid" + . + "Allocating a fresh buffer/string/bytevector on every iteration of a hot loop: a fresh 4 KB read buffer per syscall, a fresh 64 KB chunk per copy iteration, a 1-byte bytevector per byte read, one bytevector per dirent then a final copy, a fresh substring per comparison position. Each is GC pressure and, in aggregate, can dominate runtime (jerboa-emacs PTY read, jerboa-signal attach copy, jerboa-ssh banner read, jerboa-fuse encode-dirents, jerboa-top/webex per-position substring searches).") + ("id" . "per-element-allocation-in-hot-loop") + ("kinds" "performance") ("pattern" . "") + ("severity" . "medium") + ("tags" "allocation" "hot-loop" "buffer-reuse" "gc-pressure" + "bytevector" "substring") + ("title" + . + "Allocating a fresh buffer/string per iteration in a hot loop") + ("tools" "jerboa_howto")) + (("advice" + . + "Hoist the invariant computation out of the loop. Memoize per-key results in a hash table (e.g. sender→color). Precompute a derived field at parse/build time and carry it in the record (e.g. a downcased command-lc field, a search_blob, a tag_count). Build a lookup structure once (line-offset index, pattern-id→meta table) instead of re-scanning per query.") + ("avoid" + . + "Recomputing loop-invariant data on every iteration or call: re-downcasing the same string per call, re-splitting the full buffer per named-block lookup, re-running string->utf8 on the entire source per node-text call, re-compiling a regex per file walked, re-computing a comparison key per comparison, re-deriving a latin1/normalized copy for each sub-buffer from the same bytevector. Each turns an O(n) or O(1) datum into a per-iteration cost (jerboa-top render re-downcase, jerboa-emacs org-babel re-split, jerboa-treesitter node-text re-utf8, jerboa-emacs eshell regex recompile, jerboa-imagesite per-media haystack downcase, jerboa-virus per-sub-buffer latin1).") + ("id" . "rederive-invariant-per-iteration") + ("kinds" "performance") ("pattern" . "") + ("severity" . "medium") + ("tags" "hoist" "memoize" "precompute" "loop-invariant" + "downcase" "cache") + ("title" + . + "Re-deriving loop-invariant data on every iteration") + ("tools" "jerboa_howto")) + (("advice" + . + "Defer such optimizations unless you can design invalidation-on-every-mutation PLUS a targeted aliasing/reuse test that proves no stale access. For on-disk formats, require versioning + a migration path before adding a field. A wrong cache in an encrypted FS is a confidentiality/integrity bug, not just a perf bug — when in doubt, leave the correct slow path and document the optimization as future work.") + ("avoid" + . + "Adding a cache or index to a crypto or on-disk-format code path without invalidation-on-mutation and targeted aliasing tests. Examples found and correctly DEFERRED in the P2 wave: caching a decrypted indirect block keyed by block number in jerboa-fuse vault.ss — but free-inode-blocks! frees+reuses block numbers, so a stale cache entry aliases → cross-file data leak/corruption in an ENCRYPTED filesystem; and adding a d_type field to a fixed-size 264-byte on-disk dirent → an on-disk format break with no versioning/migration path.") + ("id" . "optimize-crypto-or-format-without-invalidation") + ("kinds" "performance" "security" "correctness") + ("pattern" . "") ("severity" . "high") + ("tags" "cache" "invalidation" "crypto" "on-disk-format" + "aliasing" "data-leak" "defer") + ("title" + . + "Caching/indexing a crypto or on-disk-format path without invalidation + aliasing tests") + ("tools" "jerboa_security_scan"))) --- a/data/cookbooks.sexp +++ b/data/cookbooks.sexp @@ -6994,4 +6994,64 @@ "abort" "error-code") ("title" . - "Guard every foreign-callable body (prevent conditions escaping into C)"))) + "Guard every foreign-callable body (prevent conditions escaping into C)")) + (("code" + . + ";; Typed Android Intent functions are SIDE-EFFECTING (return Unit).\n;; Never chain them as expressions. Use begin blocks.\n\n;; WRONG - these return Unit, not Intent:\n;; (intentAddFlagsRaw (intentDataSetRaw (intentForAction \"VIEW\") uri) flags)\n\n;; RIGHT - create intent, mutate in begin block:\n(def (openPdf (activity : MainActivity) (file : File))\n :\n Unit\n (let ([intent (intentForAction \"android.intent.action.VIEW\")])\n (begin\n (intentTypeSetRaw intent \"application/pdf\")\n (intentDataSetRaw intent (uriFromFileRaw file))\n (intentAddFlagsRaw intent (intentFlagGrantReadUriPermission))\n (mainActivityStartActivityRaw activity intent))))") ("id" . "typed-android-intent-side-effects") + ("imports" "(jerboa prelude)") + ("notes" + . + "In the typed Kotlin backend, intentTypeSetRaw, intentDataSetRaw, intentAddFlagsRaw all return Unit (they mutate the Intent in place via kotlin-member-set/kotlin-member-call). Chaining them as nested expressions causes argument-type-mismatch errors. Always create the Intent first, then mutate it in a begin block. This applies to ALL Android setter-style externs in the typed system.") + ("tags" "typed" "android" "intent" "kotlin" "side-effect" + "begin") + ("title" + . + "Typed Android: Intent functions are side-effecting (use begin blocks)")) + (("code" + . + ";; The typed Android generator copies assets from (asset-dir) in the app spec.\n;; If code reads from assets, the file MUST exist in the fixtures directory.\n\n;; In tests/fixtures/ssd-app.ss:\n(def app\n '(android-app\n (id \"com.sfb.ssdreview\")\n (asset-dir \"assets\") ;; REQUIRED - copies tests/fixtures/assets/ into APK\n ...))\n\n;; Assets directory must contain:\n;; tests/fixtures/assets/box_types.csv\n;; tests/fixtures/assets/seed_ssd_review.marker\n;; tests/fixtures/assets/seed_ssd_review.zip\n\n;; Without this, the app crashes at startup with:\n;; java.io.FileNotFoundException: box_types.csv\n;; at BoxTypeAndroidKt.boxTypesCsvText") ("id" . "typed-android-asset-dir-required") + ("imports" "(jerboa prelude)") + ("notes" + . + "The jandroid.ss generator supports (asset-dir \"path\") in the app spec. The path is relative to the spec file's directory. Files are copied to app/src/main/assets/ in the build output. If your typed code calls assetManager.open(\"filename\"), that file MUST be in the asset-dir. Missing assets cause FileNotFoundException at runtime, NOT at compile time. Always test on the emulator to catch this.") + ("tags" "typed" "android" "assets" "generator" "crash" + "fixture") + ("title" + . + "Typed Android: asset-dir in app spec is required for runtime assets")) + (("code" + . + "#!/bin/bash\n# MANDATORY emulator test after ANY typed Android code change\nADB=~/Library/Android/sdk/platform-tools/adb\n\n# 1. Generate + compile\nmake ssd-compile || exit 1\n\n# 2. Build APK\nexport ANDROID_HOME=\"$HOME/Library/Android/sdk\"\nexport JDK_HOME=$(scripts/verified-jdk.sh --home)\nexport JAVA_HOME=\"$JDK_HOME\"\nscripts/verified-gradle.sh --no-daemon -p build/ssd-security assembleDebug || exit 1\n\n# 3. Install + launch\n$ADB install -r build/ssd-security/app/build/outputs/apk/debug/app-debug.apk || exit 1\n$ADB logcat -c\n$ADB shell am force-stop com.sfb.ssdreview\n$ADB shell am start -n com.sfb.ssdreview/.MainActivity\nsleep 4\n\n# 4. Check for crashes (MUST be empty)\nCRASH=$($ADB logcat -d -s AndroidRuntime:E | grep -c FATAL)\nif [ \"$CRASH\" -gt 0 ]; then\n echo \"FAIL: App crashed on launch\"\n $ADB logcat -d -s AndroidRuntime:E | tail -30\n exit 1\nfi\n\n# 5. Verify activity is resumed\n$ADB shell dumpsys activity activities | grep -q \"topResumedActivity.*ssdreview\" || exit 1\n\necho \"PASS: App launches and runs without crash\"") ("id" . "typed-android-emulator-test-workflow") + ("imports" "") + ("notes" + . + "The human must NEVER be in the testing loop. Every code change must be verified on the Android emulator before deployment. If no emulator is running, start one with: ~/Library/Android/sdk/emulator/emulator -avd sfb_api35_arm64 -no-window -no-audio & then $ADB wait-for-device. For functional testing, use: $ADB shell uiautomator dump /sdcard/ui.xml to get the UI hierarchy, then $ADB shell input tap X Y to interact. The app is NOT done until it passes emulator testing.") + ("tags" "typed" "android" "emulator" "adb" "test" "workflow" + "mandatory") + ("title" + . + "Typed Android: mandatory emulator test workflow (no human in loop)")) + (("code" + . + "(import (jerboa prelude))\n\n;; O(1) enqueue, amortized O(1) dequeue FIFO queue.\n;; Avoids (append q (list x)) which is O(n) per enqueue -> O(n^2) over a burst\n;; (jerboa-ssh channel data queue, jerboa-signal notification queue).\n(defstruct queue (front back) #:mutable)\n\n(def (make-queue) (make-queue '() '()))\n\n(def (queue-empty? q)\n (and (null? (queue-front q)) (null? (queue-back q))))\n\n;; enqueue: cons onto back, O(1)\n(def (queue-push! q x)\n (set-queue-back! q (cons x (queue-back q))))\n\n;; dequeue: pop from front; when empty, reverse back into front (amortized O(1))\n(def (queue-pop! q)\n (when (null? (queue-front q))\n (set-queue-front! q (reverse (queue-back q)))\n (set-queue-back! q '()))\n (and (pair? (queue-front q))\n (let ([x (car (queue-front q))])\n (set-queue-front! q (cdr (queue-front q)))\n x)))\n\n(def (queue-length q)\n (+ (length (queue-front q)) (length (queue-back q))))") ("id" . "amortized-two-list-fifo-queue") + ("imports" "(jerboa prelude)") + ("notes" + . + "The classic two-list queue: enqueue conses onto the back list (O(1)); dequeue pops from the front, and when the front is empty reverses the back into the front (amortized O(1) because each element is reversed at most once). Replaces the (append q (list x)) idiom that is O(n) per enqueue and O(n^2) over a burst -- the single most common perf bug in the P2 wave (ssh per-packet channel queue, signal per-notification queue, smtp/dns/webex append-per-item). Each element is consed once and reversed once, so total work for n enqueue+dequeue is O(n).") + ("tags" "queue" "fifo" "amortized" "append" "performance" + "cons-reverse") + ("title" + . + "Amortized O(1) FIFO queue (two-list cons+reverse)")) + (("code" + . + "(import (jerboa prelude))\n\n;; Build a vector of line-start offsets once per source; binary-search for\n;; offset->line/col. Replaces an O(n) full-source scan PER lookup -- with 50k\n;; findings on a large source that is O(n*findings) (a hang); with an index it\n;; is O(n + findings*log n). (jerboa-semgrep source/offsets.ss: 50k lookups on\n;; 2 MB went from a hang to ~13 ms.)\n(def (build-line-index str)\n ;; vector element i = char offset where line i starts; line 0 starts at 0\n (let ([n (string-length str)])\n (let loop ([i 0] [starts '(0)])\n (if (>= i n)\n (list->vector (reverse starts))\n (loop (+ i 1)\n (if (char=? (string-ref str i) #\\newline)\n (cons (+ i 1) starts)\n starts))))))\n\n(def (offset->line-col line-index off)\n ;; greatest line-start index whose start <= off, via binary search\n (let ([n (vector-length line-index)])\n (let search ([lo 0] [hi (- n 1)] [ans 0])\n (if (> lo hi)\n (values ans (- off (vector-ref line-index ans))) ; line, col\n (let ([mid (quotient (+ lo hi) 2)])\n (if (<= (vector-ref line-index mid) off)\n (search (+ mid 1) hi mid)\n (search lo (- mid 1) ans)))))))") ("id" . "line-offset-index-binary-search") + ("imports" "(jerboa prelude)") + ("notes" + . + "Build the index once per source (O(n)), then each offset->line/col lookup is O(log n) via binary search for the greatest line-start <= off. Use a weak/eq hashtable keyed by the source (or source identity) to cache the index across many lookups for the same source. For byte offsets on non-ASCII source, build the index over byte offsets (or combine with a byte<->char offset map). Highest-value P2 fix: jerboa-semgrep's per-finding offset->line-col was an O(n) full-source scan, so 50000 findings on an 8 MB file was O(n*findings) and hung; the index made it ~13 ms.") + ("tags" "line-index" "binary-search" "offset" "performance" + "index" "lookup") + ("title" + . + "Line-offset index + binary search for O(log n) offset->line/col")))