Remove Conway-specific scaffold
ober
475b9da7096b00217538edfae1522fb8bc534ea3
--- a/docs/local-model-smartness-plan.md +++ b/docs/local-model-smartness-plan.md @@ -23,8 +23,8 @@ pattern repeatedly: `old_string` and `new_string` with native `edit`. - It obeys concrete verifier output more reliably than prose advice. -The Conway Game of Life task became successful only after the harness did more -of the work: +The Conway Game of Life task exposed two separate facts that must not be +confused: - `jcode verified` called `jerboa_request_advisor` before the first provider turn. @@ -34,9 +34,14 @@ of the work: - Native edit accepted `old_string` and `new_string` aliases. - Passing verifies stopped calling the failure advisor. +- A previous scaffold experiment crossed the line by adding a benchmark-specific + Conway/blinker template to `jcode`. That is not a valid harness improvement + and must not be repeated. + That is the central lesson: local models improve when the system is more deterministic, more structured, and less dependent on the model reading long -instructions correctly. +instructions correctly. They do not improve, in any meaningful measurement +sense, when benchmark solutions are encoded in the harness. ## North Star @@ -49,8 +54,8 @@ For a Jerboa task, the ideal flow is: classify task choose verifier choose write scope -load task-specific recipes and anti-patterns -choose a small verified scaffold +load domain recipes and anti-patterns +choose a small generic verified scaffold ask model to fill or repair limited regions run verifier map failure to a repair action @@ -195,9 +200,9 @@ Example: (tags . ("jerboa" "script" "vectors" "syntax-guard" "game-of-life")) (cwd_mode . "tempdir") (files . ()) - (task . "Write a text-based Conway's Game of Life in Jerboa Scheme in life.ss...") + (task . "Write a text-based Conway's Game of Life in Jerboa Scheme in life.ss. Use a 5x5 vertical blinker, advance one generation, render only the resulting board, then print PASS.") (write_scope . "life.ss") - (verify . "/Users/user/mine/jerboa/.chez/bin/scheme --libdirs /Users/user/mine/jerboa/lib --script life.ss | rg -q '^PASS$'") + (verify . "/Users/user/mine/jerboa/.chez/bin/scheme --libdirs /Users/user/mine/jerboa/lib --script life.ss > life.out && printf '.....\n.....\n.###.\n.....\n.....\nPASS\n' > expected.out && diff -u expected.out life.out") (timeout_seconds . 300) (expected . "pass")) ``` @@ -220,7 +225,7 @@ Create at least these tasks: 4. `jerboa-life-text` - Conway Game of Life with blinker. - - Verifier checks `PASS`. + - Verifier diffs the exact horizontal blinker output and `PASS`. 5. `jerboa-repair-unbalanced-existing` - Seed an existing `.ss` file with an extra close paren. @@ -1494,6 +1499,13 @@ Mitigation: - Do not add large unverified cookbook prose dumps. - Do not mark success without the configured verifier passing. - Do not optimize only for one Conway task. +- Do not add benchmark-specific templates, keyword shortcuts, or hidden solution + code to `jcode` or `jerboa-mcp`. Scaffolds must be generic patterns such as a + minimal script, CLI argument parsing, vector storage, module layout, or a test + harness skeleton. +- Do not use a verifier that only checks for `PASS` on synthesis tasks where a + generic scaffold could print `PASS` without implementing the requested + behavior. Verifiers must check task-specific observable behavior. ## Definition of Done --- a/eval/local-model/tasks.sexp +++ b/eval/local-model/tasks.sexp @@ -25,9 +25,9 @@ (tags . ("jerboa" "script" "vectors" "syntax-guard" "game-of-life" "scaffold")) (cwd_mode . "tempdir") (files . ()) - (task . "Write a text-based Conway's Game of Life in Jerboa Scheme in life.ss. Use a blinker oscillator as the built-in check. The script should render text output and print PASS on its own line if one generation turns the vertical blinker into a horizontal blinker.") + (task . "Write a text-based Conway's Game of Life in Jerboa Scheme in life.ss. Use a 5x5 grid with a vertical blinker initially alive at (2,1), (2,2), and (2,3). Advance exactly one generation, render only the resulting board using . for dead cells and # for live cells, then print PASS on its own line. The exact output should be five board rows followed by PASS, with the middle row .###..") (write_scope . "life.ss") - (verify . "/Users/user/mine/jerboa/.chez/bin/scheme --libdirs /Users/user/mine/jerboa/lib --script life.ss | rg -q '^PASS$'") + (verify . "/Users/user/mine/jerboa/.chez/bin/scheme --libdirs /Users/user/mine/jerboa/lib --script life.ss > life.out && printf '.....\n.....\n.###.\n.....\n.....\nPASS\n' > expected.out && diff -u expected.out life.out") (timeout_seconds . 300) (expected . "pass")) --- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -2425,10 +2425,6 @@ (def (normal-script-kind kind) (let ((k (string-downcase (or kind "minimal-pass")))) (cond - ((or (string-contains k "life") - (string-contains k "conway") - (string-contains k "blinker")) - "life-blinker") ((or (string-contains k "cli") (string-contains k "arg") (string-contains k "sum")) @@ -2505,104 +2501,9 @@ "\n" "(main)\n")) -(def (jerboa-life-blinker-script) - (string-append - "(import (jerboa prelude))\n" - "\n" - "(define width 5)\n" - "(define height 5)\n" - "\n" - "(define (idx x y)\n" - " (+ x (* y width)))\n" - "\n" - "(define (make-grid cells)\n" - " (let ((grid (make-vector (* width height) #f)))\n" - " (let loop ((rest cells))\n" - " (if (null? rest)\n" - " grid\n" - " (let ((cell (car rest)))\n" - " (vector-set! grid (idx (car cell) (cadr cell)) #t)\n" - " (loop (cdr rest)))))))\n" - "\n" - "(define (alive? grid x y)\n" - " (and (>= x 0)\n" - " (< x width)\n" - " (>= y 0)\n" - " (< y height)\n" - " (vector-ref grid (idx x y))))\n" - "\n" - "(define (neighbor-count grid x y)\n" - " (let y-loop ((dy -1) (total 0))\n" - " (if (> dy 1)\n" - " total\n" - " (let x-loop ((dx -1) (subtotal total))\n" - " (if (> dx 1)\n" - " (y-loop (+ dy 1) subtotal)\n" - " (x-loop (+ dx 1)\n" - " (if (and (not (and (= dx 0) (= dy 0)))\n" - " (alive? grid (+ x dx) (+ y dy)))\n" - " (+ subtotal 1)\n" - " subtotal)))))))\n" - "\n" - "(define (next-cell grid x y)\n" - " (let ((n (neighbor-count grid x y))\n" - " (a (alive? grid x y)))\n" - " (or (= n 3)\n" - " (and a (= n 2)))))\n" - "\n" - "(define (step grid)\n" - " (let ((out (make-vector (* width height) #f)))\n" - " (let y-loop ((y 0))\n" - " (if (>= y height)\n" - " out\n" - " (begin\n" - " (let x-loop ((x 0))\n" - " (if (< x width)\n" - " (begin\n" - " (vector-set! out (idx x y) (next-cell grid x y))\n" - " (x-loop (+ x 1)))\n" - " #t))\n" - " (y-loop (+ y 1)))))))\n" - "\n" - "(define (same-grid? a b)\n" - " (let loop ((i 0))\n" - " (or (= i (* width height))\n" - " (and (eq? (vector-ref a i) (vector-ref b i))\n" - " (loop (+ i 1))))))\n" - "\n" - "(define (render grid)\n" - " (let y-loop ((y 0))\n" - " (if (< y height)\n" - " (begin\n" - " (let x-loop ((x 0))\n" - " (if (< x width)\n" - " (begin\n" - " (display (if (alive? grid x y) #\\# #\\.))\n" - " (x-loop (+ x 1)))\n" - " #t))\n" - " (newline)\n" - " (y-loop (+ y 1)))\n" - " #t)))\n" - "\n" - "(define (main)\n" - " (let* ((initial (make-grid '((2 1) (2 2) (2 3))))\n" - " (expected (make-grid '((1 2) (2 2) (3 2))))\n" - " (actual (step initial)))\n" - " (render actual)\n" - " (if (same-grid? actual expected)\n" - " (begin\n" - " (display \"PASS\")\n" - " (newline))\n" - " (begin\n" - " (display \"FAIL\")\n" - " (newline)))))\n" - "\n" - "(main)\n")) - (def (jerboa-script-template kind) (let ((k (normal-script-kind kind))) (cond - ((string=? k "life-blinker") (jerboa-life-blinker-script)) ((string=? k "cli-two-args") (jerboa-cli-two-args-script)) ((string=? k "vector-grid") (jerboa-vector-grid-script)) (else (jerboa-minimal-pass-script))))) @@ -2616,7 +2517,7 @@ (cond ((not path) (error 'create_verified_jerboa_script - "missing path arg; use {\"path\":\"life.ss\",\"kind\":\"life-blinker\"}")) + "missing path arg; use {\"path\":\"main.ss\",\"kind\":\"minimal-pass\"}")) ((not (source-ss-path? path)) (error 'create_verified_jerboa_script "path must end in .ss for a Jerboa script: ~a" path)) @@ -2716,7 +2617,7 @@ (create-script-def (make-tool-def (make-tool-spec "create_verified_jerboa_script" - "Create a small executable Jerboa .ss script scaffold through the verified edit path. args: {\"path\": string ending .ss, \"kind\": \"minimal-pass\" | \"cli-two-args\" | \"vector-grid\" | \"life-blinker\"}. For new Jerboa scripts, prefer this before hand-writing a full file." + "Create a small executable Jerboa .ss script scaffold through the verified edit path. args: {\"path\": string ending .ss, \"kind\": \"minimal-pass\" | \"cli-two-args\" | \"vector-grid\"}. For new Jerboa scripts, prefer this before hand-writing a full file." *obj-schema*) (lambda (args) (create-verified-jerboa-script args cwd)) '())) @@ -2855,7 +2756,7 @@ (if run-aliases? "run/bash/shell are narrow inspection aliases only; use verify for the configured build/test command.\n" "run/bash/shell are not available in this workflow. Use verify for the configured build/test command.\n") - "For a new Jerboa .ss script, call create_verified_jerboa_script first when kind minimal-pass, cli-two-args, vector-grid, or life-blinker matches the task, then verify and only hand-edit if verify fails.\n" + "For a new Jerboa .ss script, call create_verified_jerboa_script first when a generic kind such as minimal-pass, cli-two-args, or vector-grid matches the task shape, then verify and only hand-edit if verify fails.\n" (external-tools-instruction external-tool-defs) (scope-instruction scope) (guidance-instruction task-guidance) --- a/test/run.ss +++ b/test/run.ss @@ -2886,7 +2886,7 @@ (guard (e [#t (void)]) (delete-file target-path))) (let* ([vr-dir "/tmp"] - [target "jcode-verified-scaffold-life.ss"] + [target "jcode-verified-scaffold-grid.ss"] [target-path (string-append vr-dir "/" target)] [tool-results '()] [slurp (lambda (p) (call-with-input-file p (lambda (i) (get-string-all i))))]) @@ -2900,12 +2900,12 @@ (make-wtool-call "create_verified_jerboa_script" (list (cons "path" target) - (cons "kind" "life-blinker")) + (cons "kind" "vector-grid")) #f)) (list (make-wtool-call "verify" '() #f)) - (list (make-wtool-call "done" '(("summary" . "scaffold-life-ok")) #f))))] + (list (make-wtool-call "done" '(("summary" . "scaffold-grid-ok")) #f))))] [result (parameterize ((current-write-scope scope)) - (run-workflow wf "create a life scaffold" resp + (run-workflow wf "create a vector grid scaffold" resp (list (cons 'max-iterations 8) (cons 'max-tool-errors 3) (cons 'on-message @@ -2914,18 +2914,31 @@ (set! tool-results (cons (message-content m) tool-results))))))))]) (check! "verified-run: Jerboa script scaffold reaches verified done" - result "scaffold-life-ok") - (check-pred! "verified-run: scaffold writes Conway code" + result "scaffold-grid-ok") + (check-pred! "verified-run: scaffold writes generic vector grid code" (slurp target-path) (lambda (s) - (and (str-contains? s "(define (neighbor-count") - (str-contains? s "\"PASS\"")))) + (and (str-contains? s "(define (make-grid)") + (str-contains? s "\"PASS\"") + (not (str-contains? s "neighbor-count")) + (not (str-contains? s "blinker"))))) (check-pred! "verified-run: prompt exposes scaffold tool" (workflow-system-prompt-template wf) (lambda (s) - (str-contains? s "create_verified_jerboa_script")))) + (and (str-contains? s "create_verified_jerboa_script") + (not (str-contains? s "life-blinker")))))) (guard (e [#t (void)]) (delete-file target-path))) + (let* ([source-path "src/jcode/core/verified-run.ss"] + [source (call-with-input-file source-path + (lambda (i) (get-string-all i)))]) + (check-pred! "verified-run: jcode scaffolds stay generic" + source + (lambda (s) + (and (not (str-contains? s "life-blinker")) + (not (str-contains? s "jerboa-life")) + (not (str-contains? s "neighbor-count")))))) + (let* ([vr-dir "/tmp"] [target "jcode-verified-rejected-draft-read.ss"] [target-path (string-append vr-dir "/" target)]