updates
ober
177bbbc1fd56d47a8dc0e1340b5b5f03d97adbf2
--- a/docs/FORGE.md +++ b/docs/FORGE.md @@ -33,7 +33,7 @@ see [`FORGE_PORT_PLAN.md`](FORGE_PORT_PLAN.md). | `/forge verify` | Describe + self-test the verify-gate (ATLAS verify + repair). | | `/forge bestofk` · `/forge best-of-k` | Describe + self-test best-of-k diverse generation. | | `/forge breaker` · `/forge no-progress` | Describe + self-test the no-progress loop breaker. | -| `/forge run [opts] <task>` | Verify-gated coding on the **live** model (edit → verify → done). Options: `--verify`, `--bestof`, `--cwd`, `--write-scope`. | +| `/forge run [opts] <task>` | Verify-gated coding on the **live** model (edit → verify → done). Options: `--verify`, `--bestof`, `--cwd`, `--write-scope`, `--guidance-file`, `--json`, `--status-file`. | The `workflow` / `proxy` / `ablation` / `verify` / `bestofk` / `breaker` self-tests run real code against scripted inputs (no live model), so they double @@ -52,13 +52,27 @@ HTTP endpoint. See [The proxy](#the-openai-compatible-proxy) below. ### `jcode verified` (subcommand) ```bash -jcode verified "<task>" [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] +jcode verified "<task>" [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--json] [--status-file FILE] ``` Runs the [verify-gate](#verify-gate) on the live model — edit → run `CMD` → finish only when it passes — optionally drawing the best of `K` candidates. The non-interactive twin of `/forge run`. +Use `--json` when another harness owns the terminal: stdout becomes one final +JSON object and the human trajectory moves to stderr. Use `--status-file FILE` +to write the same object to disk. `jcode verified` exits `0` only after a +passing verify and exits `1` when the run stops. The status object carries +`ok`, `status`, `verify_passed`, `exit_code`, `exit_reason`, `error_type`, +`error`, `summary`, `provider`, `model`, `verify_command`, `cwd`, and +`write_scope`. + +If MCP servers are initialized, their registered MCP-origin tools are included +in the verified workflow as structured external tools. Code changes still go +through the verified workflow's own edit tools, and success still requires the +configured `verify` command to pass. `--no-mcp` keeps those external tools out +by skipping MCP startup. + --- ## The guardrails --- a/docs/cli.md +++ b/docs/cli.md @@ -85,7 +85,7 @@ jcode connect HOST:PORT --host NAME [--token T] # controller side ### `verified` ``` -jcode verified "<task>" [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] +jcode verified "<task>" [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--json] [--status-file FILE] ``` Runs an edit→verify→done loop on the live model: it edits, runs `--verify CMD` @@ -95,6 +95,18 @@ optionally drawing the best of `K` diverse candidates. `--write-scope` accepts `--guidance-file` appends caller-supplied task context to the verified workflow prompt; use it for cookbook-generated task bundles or other external examples without baking task-specific knowledge into jcode. +When MCP is initialized, MCP-origin tools are also exposed to the verified +workflow as structured tools, so validators and project-specific MCP helpers are +available without bypassing the verified edit/verify gate. Use `--no-mcp` to +skip MCP initialization for the process. + +Automation can use `--json` to emit a single final status object on stdout; the +human trajectory is written to stderr in that mode. `--status-file FILE` writes +the same final status object to disk in both human and JSON modes. The command +exits `0` only when verification passed, and exits `1` when the verified run +stopped. Status fields include `ok`, `status`, `verify_passed`, `exit_code`, +`exit_reason`, `error_type`, `error`, `summary`, `provider`, `model`, +`verify_command`, `cwd`, and `write_scope`. See [FORGE.md](FORGE.md#verify-gate). @@ -164,7 +176,7 @@ Useful built-in workflow skill: | `/forge verify` | Describe + self-test the verify-gate. | | `/forge bestofk` · `/forge best-of-k` | Describe + self-test best-of-k generation. | | `/forge breaker` · `/forge no-progress` | Describe + self-test the no-progress breaker. | -| `/forge run [opts] <task>` | Verify-gated coding on the live model. Supports `--verify`, `--bestof`, `--cwd`, `--write-scope`, and `--guidance-file`. | +| `/forge run [opts] <task>` | Verify-gated coding on the live model. Supports `--verify`, `--bestof`, `--cwd`, `--write-scope`, `--guidance-file`, `--json`, and `--status-file`. | Each `/forge` self-test runs real code against scripted inputs (no live model), so it doubles as a smoke test. Full semantics in [FORGE.md](FORGE.md). --- a/src/jcode/core/verified-run.ss +++ b/src/jcode/core/verified-run.ss @@ -27,6 +27,7 @@ :jcode/core/verified :jcode/core/best-of-k :jcode/core/workflow-runner + :jcode/tool/registry :jcode/proxy/server) (def default-verify-command "make build") @@ -307,6 +308,63 @@ (def (arg-ref args key default) (let ((p (assoc key args))) (if p (cdr p) default))) +(def (workflow-args->hash args) + (cond + ((hash-table? args) args) + ((null? args) (make-hash-table)) + ((list? args) + (let ((ht (make-hash-table))) + (for-each + (lambda (kv) + (when (pair? kv) + (hash-put! ht (car kv) (cdr kv)))) + args) + ht)) + (else args))) + +(def *verified-reserved-tool-names* + '("read" "list" "ls" "cat" "tail" "head" "wc" "balance" + "run" "bash" "shell" + "edit" "write" "line_edit" "replace_def" "replace_range" + "verify" "done")) + +(def (schema-function schema) + (and (hash-table? schema) (hash-get schema "function"))) + +(def (workflow-mcp-tool-defs) + (filter-map + (lambda (schema) + (let* ((fn (schema-function schema)) + (name (and (hash-table? fn) (hash-get fn "name")))) + (and (string? name) + (eq? (tool-origin name) 'mcp) + (not (member name *verified-reserved-tool-names*)) + (not (write-tool-name? name)) + (let ((desc (or (hash-get fn "description") "MCP tool")) + (params (or (hash-get fn "parameters") (make-hash-table)))) + (make-tool-def + (make-tool-spec name desc params) + (lambda (args) (tool-execute name (workflow-args->hash args))) + '()))))) + (get-tool-schemas))) + +(def (external-tools-instruction tool-defs) + (if (null? tool-defs) + "" + (let* ((names (map tool-def-name tool-defs)) + (shown (take-up-to names 20)) + (more (- (length names) (length shown)))) + (string-append + "External MCP tools are available as structured tools" + (if (pair? shown) + (string-append ": " (string-join shown ", ") + (if (> more 0) + (format ", and ~a more" more) + "") + ".\n") + ".\n") + "Use MCP tools for language, API, project-specific, or validator facts before guessing. Keep code changes through edit/write/line_edit/replace_def/replace_range and final validation through verify.\n")))) + (def *tool-path-suffixes* '(".ss" ".scm" ".sls" ".md" ".txt" ".json" ".yaml" ".yml" ".c" ".h" ".rs" ".py" ".sh" ".png" ".html" ".css" ".js" @@ -807,15 +865,15 @@ (cond ((required-repair-read-block-message cwd args) => (lambda (msg) msg)) ((current-required-range-repair) (do-read-current args cwd)) - ((note-inspection-after-failed-verify! 'read) => (lambda (msg) msg)) - ((note-inspection-after-edit! 'read) => (lambda (msg) msg)) + ((note-inspection-after-failed-verify! 'read) => (lambda (msg) (error 'read msg))) + ((note-inspection-after-edit! 'read) => (lambda (msg) (error 'read msg))) (else (do-read-current args cwd)))) (def (do-list args cwd) (cond ((required-repair-tool-block-message cwd 'list args) => (lambda (msg) msg)) - ((note-inspection-after-failed-verify! 'list) => (lambda (msg) msg)) - ((note-inspection-after-edit! 'list) => (lambda (msg) msg)) + ((note-inspection-after-failed-verify! 'list) => (lambda (msg) (error 'list msg))) + ((note-inspection-after-edit! 'list) => (lambda (msg) (error 'list msg))) (else (let* ((path (or (arg-path args #f) ".")) (p (abs-path cwd path))) @@ -836,13 +894,16 @@ (string-append (string-join shown "\n") suffix))))))))) (def (shell-unavailable _args) - (or (note-inspection-after-failed-verify! 'run) - (string-append - "run/bash/shell are guidance-only in this verified workflow. Do not retry shell. Use list/read/balance for inspection, edit/line_edit/replace_def/replace_range for repairs, and verify to run the configured build/test command." - (let ((detail (current-last-verify-detail))) - (if detail - (string-append "\n\nLast verify failure:\n" (tail-lines detail 20)) - ""))))) + (cond + ((note-inspection-after-failed-verify! 'run) + => (lambda (msg) (error 'run msg))) + (else + (string-append + "run/bash/shell are guidance-only in this verified workflow. Do not retry shell. Use list/read/balance for inspection, edit/line_edit/replace_def/replace_range for repairs, and verify to run the configured build/test command." + (let ((detail (current-last-verify-detail))) + (if detail + (string-append "\n\nLast verify failure:\n" (tail-lines detail 20)) + "")))))) (def (best-repair-candidate-text cwd repair label) (let ((span (repair-span-content cwd repair))) @@ -1297,7 +1358,7 @@ (if repair (required-repair-balance-message cwd repair) "balance: missing path"))) - ((note-inspection-after-edit! 'balance) => (lambda (msg) msg)) + ((note-inspection-after-edit! 'balance) => (lambda (msg) (error 'balance msg))) ((rejected-draft-balance-message cwd path) => (lambda (msg) msg)) ((current-required-range-repair) => (lambda (repair) (required-repair-balance-message cwd repair))) @@ -1551,7 +1612,7 @@ ((cat-heredoc-write cmd cwd) => (lambda (result) result)) (else (cond - ((note-inspection-after-failed-verify! 'run) => (lambda (msg) msg)) + ((note-inspection-after-failed-verify! 'run) => (lambda (msg) (error 'run msg))) ((pending-ss-create-repair-message cwd) => (lambda (msg) msg)) ((and (string? cmd) (not (null? words)) @@ -2144,8 +2205,11 @@ (scope (or (opt-get o 'write-scope) (current-write-scope))) (run-aliases? (let ((p (assoc 'run-aliases? o))) (if p (cdr p) #t))) + (external-tools? (let ((p (assoc 'external-tools? o))) + (if p (cdr p) #t))) (terminal-on-verify? (and (opt-get o 'terminal-on-verify) #t)) - (task-guidance (opt-get o 'task-guidance))) + (task-guidance (opt-get o 'task-guidance)) + (external-tool-defs (if external-tools? (workflow-mcp-tool-defs) '()))) (let ((read-def (make-tool-def (make-tool-spec "read" @@ -2324,6 +2388,7 @@ (append (list read-def list-def ls-def cat-def tail-def head-def wc-def balance-def) (if run-aliases? (list run-def bash-def shell-def) '()) + external-tool-defs (list edit-def write-def line-edit-def replace-def replace-range-def verify-def done-def)) (if terminal-on-verify? '() (list "verify")) (if terminal-on-verify? "verify" "done") @@ -2338,6 +2403,7 @@ (if run-aliases? "run/bash/shell are narrow inspection aliases only; use verify for the configured build/test command.\n" "run/bash/shell are not available in this workflow. Use verify for the configured build/test command.\n") + (external-tools-instruction external-tool-defs) (scope-instruction scope) (guidance-instruction task-guidance) "Workflow:\n" --- a/src/jcode/mcp/client.ss +++ b/src/jcode/mcp/client.ss @@ -311,7 +311,8 @@ (let ((jcode-name (string-append prefix name))) (register-tool! jcode-name desc schema (lambda (args) - (mcp-call-tool conn name args)))))) + (mcp-call-tool conn name args))) + (set-tool-origin! jcode-name 'mcp)))) tools) ;; Cache the count so mcp-active-servers can return it without ;; doing a blocking JSON-RPC call later. --- a/src/jcode/tool/registry.ss +++ b/src/jcode/tool/registry.ss @@ -14,6 +14,8 @@ mode-blocked-message write-tool-name? register-write-tools! + set-tool-origin! + tool-origin set-tool-internal! tool-exists? tool-primary-arg @@ -72,6 +74,15 @@ (set! *write-tools* (cons n *write-tools*)))) names)) +(def (set-tool-origin! name origin) + "Tag a registered tool with its source subsystem, e.g. 'mcp." + (let ((t (hash-get *tools* name))) + (when t (hash-put! t "origin" origin)))) + +(def (tool-origin name) + (let ((t (hash-get *tools* name))) + (and t (hash-get t "origin")))) + (def (register-tool! name description schema handler) (hash-put! *tools* name (make-hash-table-from-alist --- a/src/jcode/ui/cli.ss +++ b/src/jcode/ui/cli.ss @@ -274,6 +274,8 @@ COMMANDS: [--cwd DIR] working directory (default: .). [--write-scope PATHS] all, none, or comma-separated paths. [--guidance-file FILE] add caller-supplied task context. + [--json] emit final status JSON on stdout. + [--status-file FILE] also write final status JSON to FILE. [--no-run-aliases] omit run/bash/shell inspection aliases. EXAMPLES: jcode Start interactive session @@ -741,25 +743,86 @@ EXAMPLES: (> (string-length path) 0) (read-file-string path))) -(def (run-verified-task provider task bestof vcmd cwd scope-spec run-aliases? guidance-file) - (let ((scope (parse-write-scope (or scope-spec "all")))) - (printf "Verified run (best-of-~a, verify=~a, write_scope=~a, run_aliases=~a, guidance=~a)~n task: ~a~n" - bestof (or vcmd default-verify-command) (write-scope-label scope) - (if run-aliases? "on" "off") - (if guidance-file guidance-file "none") - task) - (guard (e [#t (printf "~n✗ verified-run stopped: ~a~n" (err->string e))]) - (let* ((guidance (read-guidance-file guidance-file)) - (opt (list (cons 'best-of bestof) - (cons 'on-message vr-print-message) - (cons 'verify-command (or vcmd default-verify-command)) - (cons 'cwd (or cwd ".")) - (cons 'write-scope scope) - (cons 'run-aliases? run-aliases?) - (cons 'task-guidance guidance)))) - (let ((summary (verified-run provider task opt))) - (printf "~n✓ done: ~a~n" summary))))) - ) +(def (put-json! ht key val) + (hash-put! ht key val) + ht) + +(def (verified-status ok? task bestof verify-command cwd scope run-aliases? guidance-file status-file summary error-type error-message) + (let ((ht (make-hash-table))) + (put-json! ht "ok" ok?) + (put-json! ht "status" (if ok? "passed" "failed")) + (put-json! ht "verify_passed" ok?) + (put-json! ht "exit_code" (if ok? 0 1)) + (put-json! ht "exit_reason" (if ok? "verified" (or error-type "error"))) + (put-json! ht "error_type" (or error-type #f)) + (put-json! ht "error" (or error-message #f)) + (put-json! ht "summary" (or summary #f)) + (put-json! ht "task" task) + (put-json! ht "provider" (or (current-provider-override) (config-provider))) + (put-json! ht "model" (or (current-model-override) (config-model))) + (put-json! ht "best_of" bestof) + (put-json! ht "verify_command" verify-command) + (put-json! ht "cwd" cwd) + (put-json! ht "write_scope" (write-scope-label scope)) + (put-json! ht "run_aliases" run-aliases?) + (put-json! ht "guidance_file" (or guidance-file #f)) + (put-json! ht "status_file" (or status-file #f)) + ht)) + +(def (write-status-file! path status) + (when path + (call-with-output-file path + (lambda (out) + (display (json-object->string status) out) + (newline out)) + 'replace))) + +(def (finish-verified-status! status json? status-file) + (write-status-file! status-file status) + (when json? + (display (json-object->string status)) + (newline)) + (hash-ref status "ok" #f)) + +(def (vr-print-message-to port msg) + (parameterize ((current-output-port port)) + (vr-print-message msg))) + +(def (run-verified-task provider task bestof vcmd cwd scope-spec run-aliases? guidance-file json? status-file) + (let* ((scope (parse-write-scope (or scope-spec "all"))) + (verify-command (or vcmd default-verify-command)) + (work-cwd (or cwd ".")) + (human-port (if json? (current-error-port) (current-output-port)))) + (fprintf human-port + "Verified run (best-of-~a, verify=~a, write_scope=~a, run_aliases=~a, guidance=~a)~n task: ~a~n" + bestof verify-command (write-scope-label scope) + (if run-aliases? "on" "off") + (if guidance-file guidance-file "none") + task) + (guard (e [#t + (let* ((msg (err->string e)) + (etype (eval-error-type e)) + (status (verified-status #f task bestof verify-command work-cwd + scope run-aliases? guidance-file + status-file #f etype msg))) + (fprintf human-port "~n✗ verified-run stopped: ~a~n" msg) + (finish-verified-status! status json? status-file))]) + (let* ((guidance (read-guidance-file guidance-file)) + (opt (list (cons 'best-of bestof) + (cons 'on-message (lambda (msg) (vr-print-message-to human-port msg))) + (cons 'verify-command verify-command) + (cons 'cwd work-cwd) + (cons 'write-scope scope) + (cons 'run-aliases? run-aliases?) + (cons 'task-guidance guidance)))) + (let ((summary (parameterize ((current-output-port human-port)) + (verified-run provider task opt)))) + (fprintf human-port "~n✓ done: ~a~n" summary) + (finish-verified-status! + (verified-status #t task bestof verify-command work-cwd scope + run-aliases? guidance-file status-file + summary #f #f) + json? status-file)))))) (def (shell-words s) "Small shell-like splitter for slash-command flags. Handles whitespace, @@ -793,44 +856,49 @@ EXAMPLES: (loop (+ i 1) (cons ch cur) words #f #f #t))))))))) (def (parse-verified-options args) - (let loop ((args args) (words '()) (bestof 1) (vcmd #f) (cwd #f) (scope #f) (run-aliases? #t) (guidance-file #f)) + (let loop ((args args) (words '()) (bestof 1) (vcmd #f) (cwd #f) (scope #f) (run-aliases? #t) (guidance-file #f) (json? #f) (status-file #f)) (cond ((null? args) (let ((task (string-join (reverse words) " "))) - (values task bestof vcmd cwd scope run-aliases? guidance-file))) + (values task bestof vcmd cwd scope run-aliases? guidance-file json? status-file))) ((and (equal? (car args) "--bestof") (pair? (cdr args))) - (loop (cddr args) words (or (string->number (cadr args)) 1) vcmd cwd scope run-aliases? guidance-file)) + (loop (cddr args) words (or (string->number (cadr args)) 1) vcmd cwd scope run-aliases? guidance-file json? status-file)) ((and (equal? (car args) "--verify") (pair? (cdr args))) - (loop (cddr args) words bestof (cadr args) cwd scope run-aliases? guidance-file)) + (loop (cddr args) words bestof (cadr args) cwd scope run-aliases? guidance-file json? status-file)) ((and (equal? (car args) "--cwd") (pair? (cdr args))) - (loop (cddr args) words bestof vcmd (cadr args) scope run-aliases? guidance-file)) + (loop (cddr args) words bestof vcmd (cadr args) scope run-aliases? guidance-file json? status-file)) ((and (or (equal? (car args) "--write-scope") (equal? (car args) "--scope")) (pair? (cdr args))) - (loop (cddr args) words bestof vcmd cwd (cadr args) run-aliases? guidance-file)) + (loop (cddr args) words bestof vcmd cwd (cadr args) run-aliases? guidance-file json? status-file)) ((and (or (equal? (car args) "--guidance-file") (equal? (car args) "--guide-file") (equal? (car args) "--task-guidance")) (pair? (cdr args))) - (loop (cddr args) words bestof vcmd cwd scope run-aliases? (cadr args))) + (loop (cddr args) words bestof vcmd cwd scope run-aliases? (cadr args) json? status-file)) + ((equal? (car args) "--json") + (loop (cdr args) words bestof vcmd cwd scope run-aliases? guidance-file #t status-file)) + ((and (equal? (car args) "--status-file") (pair? (cdr args))) + (loop (cddr args) words bestof vcmd cwd scope run-aliases? guidance-file json? (cadr args))) ((equal? (car args) "--no-run-aliases") - (loop (cdr args) words bestof vcmd cwd scope #f guidance-file)) - (else (loop (cdr args) (cons (car args) words) bestof vcmd cwd scope run-aliases? guidance-file))))) + (loop (cdr args) words bestof vcmd cwd scope #f guidance-file json? status-file)) + (else (loop (cdr args) (cons (car args) words) bestof vcmd cwd scope run-aliases? guidance-file json? status-file))))) -;; `jcode verified <task> [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--no-run-aliases]` +;; `jcode verified <task> [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--json] [--status-file FILE] [--no-run-aliases]` (def (verified-main args) - (let-values (((task bestof vcmd cwd scope run-aliases? guidance-file) (parse-verified-options args))) + (let-values (((task bestof vcmd cwd scope run-aliases? guidance-file json? status-file) (parse-verified-options args))) (if (string=? task "") (begin (fprintf (current-error-port) - "[ERROR] usage: jcode verified <task> [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--no-run-aliases]~n") + "[ERROR] usage: jcode verified <task> [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--json] [--status-file FILE] [--no-run-aliases]~n") (exit 1)) - (run-verified-task (get-current-provider) task bestof vcmd cwd scope run-aliases? guidance-file)))) + (unless (run-verified-task (get-current-provider) task bestof vcmd cwd scope run-aliases? guidance-file json? status-file) + (exit 1))))) (def (handle-command input session-id) (let ((cmd (string-trim (substring input 1 (string-length input))))) (cond ((equal? cmd "help") - (display "\nCommands:\n /help Show this help\n /model [name] Show or set model\n /provider [name] Show or set provider\n /expert <prompt> Route one prompt to the configured expert model\n /plan Switch to PLAN mode (read-only)\n /build Switch to BUILD mode (read+write)\n /mode Show current mode\n /mcp Toggle MCP tools on/off\n /tools List available tools\n /agents List named sub-agent roles (task tool)\n /themes List available TUI themes\n /theme [name] Cycle or switch TUI theme\n /clear Start a new session\n /sessions List saved sessions\n /compact Show message count\n /undo [N] Revert last N checkpoint(s) (default 1)\n /checkpoints List recent shadow-git checkpoints\n /forge [on|off] Show or toggle forge guardrails\n /forge sampling <off|on|strict> Per-model sampling policy\n /forge workflow Describe + self-test the workflow engine\n /forge proxy Describe + self-test the OpenAI-compatible proxy\n /forge ablation Describe + self-test the eval/ablation harness\n /forge verify Describe + self-test the verify-gate (ATLAS verify+repair)\n /forge bestofk Describe + self-test best-of-k diverse-gen (ATLAS Phase-1)\n /forge breaker Describe + self-test the no-progress loop breaker\n /forge run [opts] <task> Verify-gated coding; opts: --verify, --bestof, --write-scope, --guidance-file, --no-run-aliases\n /quit Exit\n\nMulti-line: end a line with \\ to continue on the next line.\n\n")) + (display "\nCommands:\n /help Show this help\n /model [name] Show or set model\n /provider [name] Show or set provider\n /expert <prompt> Route one prompt to the configured expert model\n /plan Switch to PLAN mode (read-only)\n /build Switch to BUILD mode (read+write)\n /mode Show current mode\n /mcp Toggle MCP tools on/off\n /tools List available tools\n /agents List named sub-agent roles (task tool)\n /themes List available TUI themes\n /theme [name] Cycle or switch TUI theme\n /clear Start a new session\n /sessions List saved sessions\n /compact Show message count\n /undo [N] Revert last N checkpoint(s) (default 1)\n /checkpoints List recent shadow-git checkpoints\n /forge [on|off] Show or toggle forge guardrails\n /forge sampling <off|on|strict> Per-model sampling policy\n /forge workflow Describe + self-test the workflow engine\n /forge proxy Describe + self-test the OpenAI-compatible proxy\n /forge ablation Describe + self-test the eval/ablation harness\n /forge verify Describe + self-test the verify-gate (ATLAS verify+repair)\n /forge bestofk Describe + self-test best-of-k diverse-gen (ATLAS Phase-1)\n /forge breaker Describe + self-test the no-progress loop breaker\n /forge run [opts] <task> Verify-gated coding; opts: --verify, --bestof, --write-scope, --guidance-file, --json, --status-file, --no-run-aliases\n /quit Exit\n\nMulti-line: end a line with \\ to continue on the next line.\n\n")) ((equal? cmd "model") (printf "Provider: ~a~n" (or (current-provider-override) (config-provider))) (printf "Model: ~a~n" (or (current-model-override) (config-model))) @@ -977,16 +1045,16 @@ EXAMPLES: ((or (equal? cmd "forge breaker") (equal? cmd "forge no-progress")) (forge-print-breaker)) ((string-prefix? "forge run " cmd) - (let-values (((task bestof vcmd cwd scope run-aliases? guidance-file) + (let-values (((task bestof vcmd cwd scope run-aliases? guidance-file json? status-file) (parse-verified-options (shell-words (string-trim (substring cmd (string-length "forge run ") (string-length cmd))))))) (if (string=? task "") - (printf "Usage: /forge run [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--no-run-aliases] <task>~n") - (run-verified-task (get-current-provider) task bestof vcmd cwd scope run-aliases? guidance-file)))) + (printf "Usage: /forge run [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--json] [--status-file FILE] [--no-run-aliases] <task>~n") + (run-verified-task (get-current-provider) task bestof vcmd cwd scope run-aliases? guidance-file json? status-file)))) ((equal? cmd "forge run") - (printf "Usage: /forge run [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--no-run-aliases] <task>~n")) + (printf "Usage: /forge run [--bestof K] [--verify CMD] [--cwd DIR] [--write-scope PATHS] [--guidance-file FILE] [--json] [--status-file FILE] [--no-run-aliases] <task>~n")) ((or (equal? cmd "forge on") (equal? cmd "forge enforce") (equal? cmd "forge enforce on")) (forge-respond-enforced? #t) --- a/test/run.ss +++ b/test/run.ss @@ -1936,6 +1936,36 @@ (lambda (s) (str-contains? s "run/bash/shell are not available")))) +(register-tool! "jcode_test_plain_lookup" + "A plain registry tool that should not enter verified workflow." + '(("type" . "object")) + (lambda (a) "plain")) +(register-tool! "jerboa_test_lookup" + "A custom-prefixed MCP-origin lookup tool." + '(("type" . "object") + ("properties" . (("q" . (("type" . "string")))))) + (lambda (a) (json-object->string a))) +(set-tool-origin! "jerboa_test_lookup" 'mcp) +(let* ([wf (coding-workflow "true" "/tmp")] + [mcp-tool (workflow-get-tool-def wf "jerboa_test_lookup")] + [plain-tool (workflow-get-tool-def wf "jcode_test_plain_lookup")] + [disabled-wf (coding-workflow "true" "/tmp" + (list (cons 'external-tools? #f)))]) + (check! "verified-run: MCP-origin custom prefix is exposed" + (and mcp-tool #t) #t) + (check! "verified-run: plain registry tool is not exposed" + plain-tool #f) + (check! "verified-run: external tools option disables MCP bridge" + (workflow-get-tool-def disabled-wf "jerboa_test_lookup") #f) + (check-pred! "verified-run: MCP-origin tool receives converted args" + ((tool-def-callable mcp-tool) '(("q" . "life"))) + (lambda (s) (str-contains? s "\"life\""))) + (check-pred! "verified-run: prompt mentions external MCP tools" + (workflow-system-prompt-template wf) + (lambda (s) + (and (str-contains? s "External MCP tools") + (str-contains? s "jerboa_test_lookup"))))) + (let* ([wf (coding-workflow "true" "/tmp" (list (cons 'task-guidance "recipe-context-marker")))]) (check-pred! "verified-run: caller guidance appears in prompt" @@ -3030,7 +3060,9 @@ [slurp (lambda (p) (call-with-input-file p (lambda (i) (get-string-all i))))]) (guard (e [#t (void)]) (delete-file target-path)) (call-with-output-file target-path - (lambda (o) (display initial o)) + (lambda (o) + (display initial o) + (flush-output-port o)) 'replace) (let* ([scope (parse-write-scope target)] [wf (coding-workflow (string-append "grep -q fixed " target) vr-dir @@ -3073,13 +3105,14 @@ result "balance-after-reads-ok") (check! "verified-run: balance-after-reads writes fixed file" (slurp target-path) fixed) - (check-pred! "verified-run: over-inspection returns nonfatal guidance" + (check-pred! "verified-run: over-inspection counts against tool-error budget" (reverse tool-results) (lambda (xs) (let loop ([ys xs]) (cond [(null? ys) #f] - [(str-contains? (car ys) "inspection limit reached after failed verify") #t] + [(and (str-contains? (car ys) "[ToolError]") + (str-contains? (car ys) "inspection limit reached after failed verify")) #t] [else (loop (cdr ys))])))) (check-pred! "verified-run: balance result survives read budget" (reverse tool-results)