Add MCP call log and reply timing UI
ober
cc8f3989d70f879d23ace0af6595544f5a6844f0
--- a/android/app/src/main/java/dev/jerboa/jcode/JcodeClient.kt +++ b/android/app/src/main/java/dev/jerboa/jcode/JcodeClient.kt @@ -26,9 +26,23 @@ import java.util.concurrent.atomic.AtomicBoolean */ class JcodeClient(private val context: Context) { + data class McpCall( + val server: String, + val method: String, + val id: Long, + val status: String, + val startedMs: Long, + val durationMs: Long, + val tool: String, + val params: String, + val arguments: String, + val result: String, + val error: String + ) + interface Listener { fun onConnected() - fun onReady(sessionId: String, title: String) + fun onReady(sessionId: String, title: String, cwd: String) fun onHistoryUser(content: String) fun onHistoryAssistant(content: String) fun onHistoryTool(name: String, argsJson: String) @@ -36,7 +50,7 @@ class JcodeClient(private val context: Context) { fun onToken(text: String) fun onToolStart(name: String, argsJson: String) fun onToolEnd(name: String) - fun onTurnEnd(sessionId: String) + fun onTurnEnd(sessionId: String, elapsedMs: Long) fun onError(message: String) fun onCancelled() fun onDisconnected(reason: String) @@ -46,6 +60,8 @@ class JcodeClient(private val context: Context) { fun onModelsEnd() fun onUsage(tokensIn: Long, tokensOut: Long, cost: Double) fun onStatus(mcp: List<Pair<String, Int>>, lspActive: Boolean) + fun onMcpCall(call: McpCall) + fun onMcpCallsEnd() } private val main = Handler(Looper.getMainLooper()) @@ -230,6 +246,10 @@ class JcodeClient(private val context: Context) { send(JSONObject().put("type", "get_status")) } + fun sendGetMcpCalls() { + send(JSONObject().put("type", "get_mcp_calls")) + } + fun sendCancel() { send(JSONObject().put("type", "cancel")) } @@ -279,7 +299,8 @@ class JcodeClient(private val context: Context) { when (type) { "ready" -> l.onReady( obj.optString("session_id", ""), - obj.optString("session_title", "") + obj.optString("session_title", ""), + obj.optString("cwd", "") ) "history" -> { val role = obj.optString("role", "") @@ -301,7 +322,10 @@ class JcodeClient(private val context: Context) { obj.optJSONObject("args")?.toString() ?: "{}" ) "tool_end" -> l.onToolEnd(obj.optString("name", "?")) - "turn_end" -> l.onTurnEnd(obj.optString("session_id", "")) + "turn_end" -> l.onTurnEnd( + obj.optString("session_id", ""), + obj.optLong("elapsed_ms", -1) + ) "cancelled" -> l.onCancelled() "error" -> l.onError(obj.optString("message", "unknown error")) "config" -> { @@ -342,6 +366,22 @@ class JcodeClient(private val context: Context) { } l.onStatus(mcpList, obj.optBoolean("lsp", false)) } + "mcp_call" -> l.onMcpCall( + McpCall( + server = obj.optString("server", ""), + method = obj.optString("method", ""), + id = obj.optLong("id", 0), + status = obj.optString("status", ""), + startedMs = obj.optLong("started_ms", 0), + durationMs = obj.optLong("duration_ms", 0), + tool = obj.optString("tool", ""), + params = obj.optString("params", ""), + arguments = obj.optString("arguments", ""), + result = obj.optString("result", ""), + error = obj.optString("error", "") + ) + ) + "mcp_calls_end" -> l.onMcpCallsEnd() "pong" -> {} // silent heartbeat ack else -> Log.d(TAG, "unhandled event type: $type") } --- a/android/app/src/main/java/dev/jerboa/jcode/MainActivity.kt +++ b/android/app/src/main/java/dev/jerboa/jcode/MainActivity.kt @@ -25,12 +25,16 @@ import android.app.AlertDialog * over a localhost TCP socket, wires JSONL events to dynamically-appended * bubble views in a ScrollView. * - * Two tabs: Chat (user + assistant messages) and Tools (tool invocations). + * Three tabs: Chat (user + assistant messages), Tools (tool invocations), + * and MCP (local MCP JSON-RPC calls). * Token updates are batched and scroll is throttled to reduce flicker. */ class MainActivity : Activity(), JcodeClient.Listener { + private enum class Page { CHAT, TOOLS, MCP } + private lateinit var titleText: TextView + private lateinit var cwdText: TextView private lateinit var modeLabel: TextView private lateinit var modeSwitch: Switch private lateinit var btnNewSession: ImageButton @@ -39,8 +43,11 @@ class MainActivity : Activity(), JcodeClient.Listener { private lateinit var chatContainer: LinearLayout private lateinit var toolsScroll: ScrollView private lateinit var toolsContainer: LinearLayout + private lateinit var mcpScroll: ScrollView + private lateinit var mcpContainer: LinearLayout private lateinit var tabChat: TextView private lateinit var tabTools: TextView + private lateinit var tabMcp: TextView private lateinit var statusLine: TextView private lateinit var inputText: EditText private lateinit var btnSend: Button @@ -59,15 +66,17 @@ class MainActivity : Activity(), JcodeClient.Listener { /** TextView of the in-progress assistant bubble, or null when idle. */ private var currentAssistantText: TextView? = null + private var currentAssistantMeta: TextView? = null /** StringBuilder accumulating tokens for the in-progress bubble. */ private val currentAssistantBuf = StringBuilder() private var sending = false - private var chatTabActive = true + private var activePage = Page.CHAT /** Count of tool events since last tab switch — shown as badge on Tools tab. */ private var toolCount = 0 + private var mcpCount = 0 /** Provider/model state from server */ private var currentProvider = "" @@ -79,7 +88,11 @@ class MainActivity : Activity(), JcodeClient.Listener { private var scrollPending = false private val scrollRunnable = Runnable { scrollPending = false - val sv = if (chatTabActive) chatScroll else toolsScroll + val sv = when (activePage) { + Page.CHAT -> chatScroll + Page.TOOLS -> toolsScroll + Page.MCP -> mcpScroll + } sv.fullScroll(View.FOCUS_DOWN) } @@ -104,6 +117,7 @@ class MainActivity : Activity(), JcodeClient.Listener { setContentView(R.layout.activity_main) titleText = findViewById(R.id.title_text) + cwdText = findViewById(R.id.cwd_text) modeLabel = findViewById(R.id.mode_label) modeSwitch = findViewById(R.id.mode_switch) btnNewSession = findViewById(R.id.btn_new_session) @@ -112,8 +126,11 @@ class MainActivity : Activity(), JcodeClient.Listener { chatContainer = findViewById(R.id.chat_container) toolsScroll = findViewById(R.id.tools_scroll) toolsContainer = findViewById(R.id.tools_container) + mcpScroll = findViewById(R.id.mcp_scroll) + mcpContainer = findViewById(R.id.mcp_container) tabChat = findViewById(R.id.tab_chat) tabTools = findViewById(R.id.tab_tools) + tabMcp = findViewById(R.id.tab_mcp) statusLine = findViewById(R.id.status_line) inputText = findViewById(R.id.input_text) btnSend = findViewById(R.id.btn_send) @@ -131,8 +148,9 @@ class MainActivity : Activity(), JcodeClient.Listener { updateModeLabel(modeSwitch.isChecked) // Tab switching - tabChat.setOnClickListener { switchTab(chat = true) } - tabTools.setOnClickListener { switchTab(chat = false) } + tabChat.setOnClickListener { switchTab(Page.CHAT) } + tabTools.setOnClickListener { switchTab(Page.TOOLS) } + tabMcp.setOnClickListener { switchTab(Page.MCP) } btnSettings.setOnClickListener { startActivity(Intent(this, SettingsActivity::class.java)) @@ -142,9 +160,12 @@ class MainActivity : Activity(), JcodeClient.Listener { if (!client.isConnected()) return@setOnClickListener chatContainer.removeAllViews() toolsContainer.removeAllViews() + mcpContainer.removeAllViews() toolCount = 0 + mcpCount = 0 updateTabLabels() currentAssistantText = null + currentAssistantMeta = null currentAssistantBuf.setLength(0) client.sendNewSession() setStatus(getString(R.string.status_ready)) @@ -201,30 +222,38 @@ class MainActivity : Activity(), JcodeClient.Listener { // ── Tab switching ─────────────────────────────────────────────────── - private fun switchTab(chat: Boolean) { - chatTabActive = chat - if (chat) { - chatScroll.visibility = View.VISIBLE - toolsScroll.visibility = View.GONE - } else { - chatScroll.visibility = View.GONE - toolsScroll.visibility = View.VISIBLE - // Clear badge when viewing tools - toolCount = 0 - } + private fun switchTab(page: Page) { + activePage = page + chatScroll.visibility = if (page == Page.CHAT) View.VISIBLE else View.GONE + toolsScroll.visibility = if (page == Page.TOOLS) View.VISIBLE else View.GONE + mcpScroll.visibility = if (page == Page.MCP) View.VISIBLE else View.GONE + + if (page == Page.TOOLS) toolCount = 0 + if (page == Page.MCP) mcpCount = 0 updateTabLabels() + throttledScroll() } private fun updateTabLabels() { + val chatActive = activePage == Page.CHAT + val toolsActive = activePage == Page.TOOLS + val mcpActive = activePage == Page.MCP + tabChat.setTextColor(resources.getColor( - if (chatTabActive) R.color.accent else R.color.text_muted, theme)) - tabChat.setTypeface(null, if (chatTabActive) android.graphics.Typeface.BOLD else android.graphics.Typeface.NORMAL) + if (chatActive) R.color.accent else R.color.text_muted, theme)) + tabChat.setTypeface(null, if (chatActive) android.graphics.Typeface.BOLD else android.graphics.Typeface.NORMAL) - val toolsLabel = if (toolCount > 0 && chatTabActive) "Tools ($toolCount)" else "Tools" + val toolsLabel = if (toolCount > 0 && !toolsActive) "Tools ($toolCount)" else "Tools" tabTools.text = toolsLabel tabTools.setTextColor(resources.getColor( - if (!chatTabActive) R.color.accent else R.color.text_muted, theme)) - tabTools.setTypeface(null, if (!chatTabActive) android.graphics.Typeface.BOLD else android.graphics.Typeface.NORMAL) + if (toolsActive) R.color.accent else R.color.text_muted, theme)) + tabTools.setTypeface(null, if (toolsActive) android.graphics.Typeface.BOLD else android.graphics.Typeface.NORMAL) + + val mcpLabel = if (mcpCount > 0 && !mcpActive) "MCP ($mcpCount)" else "MCP" + tabMcp.text = mcpLabel + tabMcp.setTextColor(resources.getColor( + if (mcpActive) R.color.accent else R.color.text_muted, theme)) + tabMcp.setTypeface(null, if (mcpActive) android.graphics.Typeface.BOLD else android.graphics.Typeface.NORMAL) } // ── UI helpers ────────────────────────────────────────────────────── @@ -285,9 +314,12 @@ class MainActivity : Activity(), JcodeClient.Listener { private fun beginAssistant() { val view = layoutInflater.inflate(R.layout.item_chat_assistant, chatContainer, false) val tv = view.findViewById<TextView>(R.id.message_text) + val meta = view.findViewById<TextView>(R.id.reply_meta) tv.text = "" + meta.visibility = View.GONE chatContainer.addView(view) currentAssistantText = tv + currentAssistantMeta = meta currentAssistantBuf.setLength(0) throttledScroll() } @@ -314,9 +346,16 @@ class MainActivity : Activity(), JcodeClient.Listener { } } - private fun finalizeAssistant() { + private fun finalizeAssistant(elapsedMs: Long = -1) { flushTokens() + if (elapsedMs >= 0) { + currentAssistantMeta?.let { + it.text = "Reply took ${formatDuration(elapsedMs)}" + it.visibility = View.VISIBLE + } + } currentAssistantText = null + currentAssistantMeta = null currentAssistantBuf.setLength(0) } @@ -328,7 +367,42 @@ class MainActivity : Activity(), JcodeClient.Listener { toolCount++ updateTabLabels() // Auto-scroll tools pane if it's visible - if (!chatTabActive) { + if (activePage == Page.TOOLS) { + throttledScroll() + } + } + + private fun appendMcpCall(call: JcodeClient.McpCall) { + val view = layoutInflater.inflate(R.layout.item_mcp_call, mcpContainer, false) + val label = if (call.tool.isNotEmpty()) call.tool else call.method + val status = if (call.status.isNotEmpty()) call.status else "?" + view.findViewById<TextView>(R.id.mcp_header).text = + "$status ${call.server} $label" + view.findViewById<TextView>(R.id.mcp_meta).text = + "id ${call.id} · ${formatDuration(call.durationMs)}" + + val details = mutableListOf<String>() + if (call.method.isNotEmpty()) details.add("method: ${call.method}") + if (call.arguments.isNotEmpty()) { + details.add("arguments:") + details.add(call.arguments) + } else if (call.params.isNotEmpty()) { + details.add("params:") + details.add(call.params) + } + if (call.error.isNotEmpty()) { + details.add("error:") + details.add(call.error) + } else if (call.result.isNotEmpty()) { + details.add("result:") + details.add(call.result) + } + view.findViewById<TextView>(R.id.mcp_details).text = details.joinToString("\n") + + mcpContainer.addView(view) + mcpCount++ + updateTabLabels() + if (activePage == Page.MCP) { throttledScroll() } } @@ -404,8 +478,9 @@ class MainActivity : Activity(), JcodeClient.Listener { setSending(true) appendUser(text) - // Reset tool count for new turn + // Reset unread counts for the new turn. toolCount = 0 + mcpCount = 0 updateTabLabels() beginAssistant() setStatus(getString(R.string.status_thinking)) @@ -431,18 +506,22 @@ class MainActivity : Activity(), JcodeClient.Listener { setStatus("Connected") } - override fun onReady(sessionId: String, title: String) { + override fun onReady(sessionId: String, title: String, cwd: String) { // Server may follow `ready` with `history` events to repopulate the // chat. Clear stale views first so reconnecting to the same session // (banner reconnect or app relaunch) doesn't double-render messages. chatContainer.removeAllViews() toolsContainer.removeAllViews() + mcpContainer.removeAllViews() toolCount = 0 + mcpCount = 0 updateTabLabels() currentAssistantText = null + currentAssistantMeta = null currentAssistantBuf.setLength(0) if (title.isNotEmpty()) titleText.text = title + cwdText.text = shortPath(cwd) setStatus("Ready \u00b7 session ${sessionId.take(8)}") } @@ -454,6 +533,7 @@ class MainActivity : Activity(), JcodeClient.Listener { // Replay: render the full content as a non-streaming assistant bubble. val view = layoutInflater.inflate(R.layout.item_chat_assistant, chatContainer, false) view.findViewById<TextView>(R.id.message_text).text = content + view.findViewById<TextView>(R.id.reply_meta).visibility = View.GONE chatContainer.addView(view) throttledScroll() } @@ -479,8 +559,8 @@ class MainActivity : Activity(), JcodeClient.Listener { setStatus(getString(R.string.status_thinking)) } - override fun onTurnEnd(sessionId: String) { - finalizeAssistant() + override fun onTurnEnd(sessionId: String, elapsedMs: Long) { + finalizeAssistant(elapsedMs) hideStatus() setSending(false) } @@ -544,6 +624,14 @@ class MainActivity : Activity(), JcodeClient.Listener { infoConnections.text = if (parts.isEmpty()) "No MCP/LSP" else parts.joinToString(" \u00b7 ") } + override fun onMcpCall(call: JcodeClient.McpCall) { + appendMcpCall(call) + } + + override fun onMcpCallsEnd() { + if (activePage == Page.MCP) throttledScroll() + } + private fun formatCount(n: Long): String = when { n >= 1_000_000 -> String.format("%.1fM", n / 1_000_000.0) n >= 1_000 -> String.format("%.1fk", n / 1_000.0) @@ -556,6 +644,36 @@ class MainActivity : Activity(), JcodeClient.Listener { else -> String.format("$%.2f", c) } + private fun formatDuration(ms: Long): String = when { + ms < 0 -> "" + ms < 1_000 -> "${ms}ms" + ms < 60_000 -> String.format("%.1fs", ms / 1_000.0) + else -> { + val minutes = ms / 60_000 + val seconds = (ms % 60_000) / 1_000 + "${minutes}m ${seconds}s" + } + } + + private fun shortPath(path: String): String { + if (path.isEmpty()) return "" + val normalized = path.trimEnd('/').ifEmpty { "/" } + val homeMarkers = listOf("/home/", "/files/home/", "/Users/") + return when { + normalized.length <= 34 -> normalized + homeMarkers.any { normalized.contains(it) } -> { + val marker = homeMarkers.first { normalized.contains(it) } + val suffix = normalized.substringAfter(marker) + val parts = suffix.split('/').filter { it.isNotEmpty() } + if (parts.size <= 2) "~/$suffix" else "~/.../${parts.takeLast(2).joinToString("/")}" + } + else -> { + val parts = normalized.split('/').filter { it.isNotEmpty() } + if (parts.size <= 2) normalized else ".../${parts.takeLast(2).joinToString("/")}" + } + } + } + companion object { private const val TAG = "jcode.Main" } --- a/android/app/src/main/res/layout/activity_main.xml +++ b/android/app/src/main/res/layout/activity_main.xml @@ -26,6 +26,20 @@ android:textStyle="bold" /> <TextView + android:id="@+id/cwd_text" + android:layout_width="0dp" + android:layout_height="wrap_content" + android:layout_marginStart="8dp" + android:layout_marginEnd="8dp" + android:layout_weight="1" + android:ellipsize="start" + android:gravity="end" + android:maxLines="1" + android:text="" + android:textColor="@color/text_muted" + android:textSize="11sp" /> + + <TextView android:id="@+id/mode_label" android:layout_width="wrap_content" android:layout_height="wrap_content" @@ -168,7 +182,7 @@ android:textSize="10sp" /> </LinearLayout> - <!-- Tab bar: Chat / Tools --> + <!-- Tab bar: Chat / Tools / MCP --> <LinearLayout android:layout_width="match_parent" android:layout_height="40dp" @@ -202,6 +216,22 @@ android:textColor="@color/text_muted" android:textSize="14sp" android:background="?android:attr/selectableItemBackground" /> + + <View + android:layout_width="1dp" + android:layout_height="match_parent" + android:background="@color/bg_input" /> + + <TextView + android:id="@+id/tab_mcp" + android:layout_width="0dp" + android:layout_height="match_parent" + android:layout_weight="1" + android:gravity="center" + android:text="MCP" + android:textColor="@color/text_muted" + android:textSize="14sp" + android:background="?android:attr/selectableItemBackground" /> </LinearLayout> <!-- Chat transcript (visible when Chat tab selected) --> @@ -241,6 +271,25 @@ android:orientation="vertical" /> </ScrollView> + <!-- MCP call log (visible when MCP tab selected) --> + <ScrollView + android:id="@+id/mcp_scroll" + android:layout_width="match_parent" + android:layout_height="0dp" + android:layout_weight="1" + android:fillViewport="true" + android:paddingTop="8dp" + android:paddingBottom="8dp" + android:scrollbars="vertical" + android:visibility="gone"> + + <LinearLayout + android:id="@+id/mcp_container" + android:layout_width="match_parent" + android:layout_height="wrap_content" + android:orientation="vertical" /> + </ScrollView> + <!-- Status line (shown during tool calls / errors) --> <TextView android:id="@+id/status_line" --- a/android/app/src/main/res/layout/item_chat_assistant.xml +++ b/android/app/src/main/res/layout/item_chat_assistant.xml @@ -18,4 +18,18 @@ android:textColor="@color/text_primary" android:textIsSelectable="true" android:textSize="13sp" /> + + <TextView + android:id="@+id/reply_meta" + android:layout_width="match_parent" + android:layout_height="wrap_content" + android:background="@color/bubble_assistant" + android:fontFamily="monospace" + android:paddingStart="12dp" + android:paddingEnd="12dp" + android:paddingTop="0dp" + android:paddingBottom="10dp" + android:textColor="@color/text_muted" + android:textSize="10sp" + android:visibility="gone" /> </LinearLayout> new file mode 100644 --- /dev/null +++ b/android/app/src/main/res/layout/item_mcp_call.xml @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="utf-8"?> +<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" + android:layout_width="match_parent" + android:layout_height="wrap_content" + android:orientation="vertical" + android:paddingStart="16dp" + android:paddingEnd="16dp" + android:paddingTop="4dp" + android:paddingBottom="4dp"> + + <TextView + android:id="@+id/mcp_header" + android:layout_width="match_parent" + android:layout_height="wrap_content" + android:background="@color/bubble_tool" + android:fontFamily="monospace" + android:paddingStart="10dp" + android:paddingEnd="10dp" + android:paddingTop="8dp" + android:paddingBottom="2dp" + android:textColor="@color/accent" + android:textSize="11sp" + android:textStyle="bold" /> + + <TextView + android:id="@+id/mcp_meta" + android:layout_width="match_parent" + android:layout_height="wrap_content" + android:background="@color/bubble_tool" + android:fontFamily="monospace" + android:paddingStart="10dp" + android:paddingEnd="10dp" + android:paddingTop="0dp" + android:paddingBottom="4dp" + android:textColor="@color/text_muted" + android:textSize="10sp" /> + + <TextView + android:id="@+id/mcp_details" + android:layout_width="match_parent" + android:layout_height="wrap_content" + android:background="@color/bubble_tool" + android:fontFamily="monospace" + android:paddingStart="10dp" + android:paddingEnd="10dp" + android:paddingTop="0dp" + android:paddingBottom="8dp" + android:textColor="@color/text_secondary" + android:textIsSelectable="true" + android:textSize="10sp" /> +</LinearLayout> --- a/docs/FORGE.md +++ b/docs/FORGE.md @@ -166,10 +166,12 @@ leaving genuine edit/verify/repair cycles untouched. A plain ### Expert escalation `core/escalation.ss` + `core/expert.ss` escalate a hard step to a stronger -"expert" model when the local model's **confidence** is low. Confidence comes -from streaming logprobs — mean/min logprob and token entropy. Logprob requests -are opt-in (`expert.escalation.request_logprobs: true` in config, which sets -`logprobs: true, top_logprobs: 5` on the call). In the [TUI](tui.md) an +"expert" model when the local model appears stuck or low-confidence. Signals +include repeated tool loops, repeated tool errors, no-progress tool results, +truncation, and optional token-confidence stats (`mean_logprob` and +`mean_entropy`) from provider logprobs. See +[Confidence signals & escalation](escalation.md) for the logprob settings, +thresholds, provider caveats, and tuning notes. In the [TUI](tui.md) an escalation renders as a distinct purple-bordered block so you can see when the expert was consulted. --- a/docs/README.md +++ b/docs/README.md @@ -24,6 +24,9 @@ elevator pitch. the workflow engine, **verify-gate**, **best-of-k**, the **no-progress breaker**, expert escalation, the OpenAI-compatible **proxy**, and the deterministic **eval/ablation** harness. +- **[Confidence signals & escalation](escalation.md)** — `request_logprobs`, + `min_mean_logprob`, `max_mean_entropy`, provider support, and how those + signals route uncertain turns to the expert model. - **[Forge port plan](FORGE_PORT_PLAN.md)** — the original design plan, kept for rationale and forge-source provenance. (Status: shipped.) new file mode 100644 --- /dev/null +++ b/docs/escalation.md @@ -0,0 +1,204 @@ +# Confidence Signals & Expert Escalation + +`jcode` can route a hard or suspicious turn from the primary model to a +configured expert model. The escalation layer combines behavioral signals +(loops, repeated failures, truncation) with optional token-confidence signals +from provider logprobs. + +This page explains the logprob settings, how they relate to "the model is +probably guessing", and how they fit into the expert-escalation feature. + +## Configuration + +Expert escalation is configured under the top-level `expert` block. The expert +model is only used when both `expert.provider` and `expert.model` are set. + +```json +{ + "provider": "mlx", + "model": "qwen3-coder-30b", + "expert": { + "provider": "openrouter", + "model": "anthropic/claude-sonnet-4", + "escalation": { + "request_logprobs": true, + "min_mean_logprob": -2.5, + "max_mean_entropy": 1.3 + } + } +} +``` + +All escalation thresholds live under `expert.escalation`. Setting a threshold to +`false` disables that signal. The logprob-based thresholds are disabled by +default because providers do not all expose logprob data. + +## `request_logprobs` + +`request_logprobs` asks the provider to return token probability metadata for +the model's generated tokens. + +For OpenAI-compatible providers, `jcode` sends: + +```json +{ + "logprobs": true, + "top_logprobs": 5 +} +``` + +The response may then include: + +- `logprob`: the log probability of the token the model actually chose. +- `top_logprobs`: the provider's top token alternatives and their log + probabilities. + +`jcode` uses these fields to compute: + +- `mean_logprob`: average chosen-token logprob across the reply. +- `min_logprob`: worst chosen-token logprob in the reply. +- `mean_entropy`: average uncertainty across the top token alternatives. + +If the provider does not return logprobs, these stats are `false`/missing and +the logprob-based escalation checks do not fire. + +## `min_mean_logprob` + +`min_mean_logprob` is a low-confidence threshold over the reply's average chosen +token logprob. + +Logprob is the natural logarithm of probability. It is usually `<= 0`: + +| Logprob | Approximate probability | Interpretation | +|---:|---:|---| +| `0.0` | `100%` | Certain, in the model's own distribution | +| `-0.1` | `90%` | Very likely | +| `-1.0` | `37%` | Plausible | +| `-2.5` | `8%` | Weak confidence | +| `-5.0` | `0.7%` | Very unlikely | + +Because logprobs are negative, lower is worse. If `min_mean_logprob` is `-2.5`, +then a reply with `mean_logprob < -2.5` is flagged as low confidence. + +This is useful for catching answers where the model repeatedly picked tokens it +itself considered unlikely. That often correlates with guessing, but it is not a +proof that the answer is wrong. + +## `max_mean_entropy` + +`max_mean_entropy` is an uncertainty threshold over the model's top token +alternatives. + +Entropy measures how spread out the alternatives are: + +- Low entropy: one token was clearly preferred. +- High entropy: several tokens were similarly plausible. + +High entropy means the model was often "at a fork". This can happen when the +prompt is ambiguous, the model is undertrained for the topic, the answer needs +external facts, or the model is trying to continue a fragile format such as code +or JSON. + +`jcode` computes entropy from `top_logprobs`, currently requesting the top 5 +alternatives. With top 5 alternatives and natural-log entropy, the maximum +renormalized entropy is `ln(5) = 1.61`. That means a threshold like `2.0` is too +high for the current `top_logprobs: 5` implementation. Practical starting +values are usually around `1.2` to `1.5`, then tuned from observed runs. + +## Relationship to correctness + +Logprob and entropy are confidence signals, not truth signals. + +They can help identify replies where the model is more likely to be guessing: + +- low `mean_logprob`: the model's chosen tokens were low probability; +- high `mean_entropy`: the model had many close alternatives; +- both together: the model was uncertain and made low-confidence choices. + +But they do not prove correctness or incorrectness: + +- A low-confidence answer can still be right. +- A high-confidence answer can still be false. +- Token confidence is about next-token prediction, not factual truth. +- Formatting, rare identifiers, unfamiliar file paths, and code tokens can + lower logprob even when the model is doing useful work. + +Treat these metrics as a routing signal: "this turn deserves stronger +verification or an expert model." Do not treat them as a factuality score. + +## Automatic escalation signals + +`jcode` escalates when any configured signal fires: + +| Signal | Meaning | +|---|---| +| `identical-tool-loop` | The assistant repeated the same tool call and arguments several times. | +| `no-text-rounds` | The assistant produced repeated empty text turns. | +| `tool-error-streak` | Several consecutive tool results looked like errors. | +| `no-progress` | Several tool results canonicalized to the same output. | +| `low-mean-logprob` | `mean_logprob` was below `min_mean_logprob`. | +| `high-mean-entropy` | `mean_entropy` was above `max_mean_entropy`. | +| `truncated-response` | The provider stopped because the reply hit the token cap. | + +The primary model can also explicitly request the expert by emitting: + +```text +<expert/> +``` + +When escalation happens, `jcode` sends the conversation to the configured expert +model with a short handoff note explaining why the primary model was escalated. +The expert response replaces the primary response. If the expert call fails, +`jcode` falls back to the primary response instead of dropping a usable answer. + +## Provider availability + +`jcode` requests and parses logprobs on its OpenAI-compatible provider path. +That includes OpenAI-compatible cloud providers as well as local `mlx` and +`ollama` when they expose compatible fields. + +Current Ollama documentation lists `logprobs` and `top_logprobs` support on its +native `/api/chat` and `/api/generate` APIs, and lists logprobs as supported on +the OpenAI-compatible `/v1/chat/completions` API. Since `jcode` uses Ollama's +OpenAI-compatible endpoint, Ollama can provide the same kind of confidence data +as MLX when the installed Ollama version returns those fields. + +Anthropic streaming does not expose token logprobs through the path `jcode` +uses, so only non-logprob escalation signals apply there. Google currently uses +a non-streaming fallback without confidence stats. Grok only provides these +signals when its backend is the OpenAI-compatible chat-completions path, not the +Responses API path. + +If a backend ignores logprob options, the stats remain unavailable and the +thresholds do not fire. If a backend rejects the options, disable +`request_logprobs` for that provider/session. + +## Tuning + +Start conservative: + +```json +{ + "expert": { + "provider": "openrouter", + "model": "anthropic/claude-sonnet-4", + "escalation": { + "request_logprobs": true, + "min_mean_logprob": -2.5, + "max_mean_entropy": 1.3 + } + } +} +``` + +Then adjust based on observed behavior: + +- Too many expert calls: lower sensitivity, for example `min_mean_logprob: + -3.0` or `max_mean_entropy: 1.5`. +- Missed obvious uncertainty: raise sensitivity, for example + `min_mean_logprob: -2.0` or `max_mean_entropy: 1.1`. +- Provider rejects requests: set `request_logprobs: false`. + +The right threshold depends on provider calibration, model family, sampling +settings, and how much extra latency/cost you are willing to spend on expert +calls. --- a/docs/remote.md +++ b/docs/remote.md @@ -31,12 +31,17 @@ One authenticated client at a time. After auth it's the same event stream as stdio: **Client → server:** `user` (`{text, mode:"plan"|"build"}`), `new_session`, -`switch_session`, `list_sessions`, `cancel`, `ping`. +`switch_session`, `list_sessions`, `get_mcp_calls`, `cancel`, `ping`. **Server → client:** `ready`, `history` / `history_tool` / `history_end` (replayed on connect), `token` (streaming text delta), `tool_start` / -`tool_end`, `turn_end`, `cancelled`, `error`, `session` / `sessions_end`, -`pong`. +`tool_end`, `mcp_call` / `mcp_calls_end`, `turn_end`, `cancelled`, `error`, +`session` / `sessions_end`, `pong`. + +`ready` includes `cwd`, the server-side working directory. `turn_end` includes +`elapsed_ms`, measured from receipt of the user event until the worker finishes +or is cancelled. `mcp_call` replays and streams local MCP JSON-RPC requests with +server, method/tool, status, argument/result previews, and `duration_ms`. Mode is per-turn — a `user` event with `mode:"plan"` hides the mutating tools for that turn. `cancel` aborts the in-flight turn immediately (the orphaned --- a/docs/tools.md +++ b/docs/tools.md @@ -111,6 +111,12 @@ built-ins. A server that fails to start is logged and skipped without blocking the others. Toggle all MCP tools at runtime with `/mcp`, or disable them for a run with `--no-mcp`. +In serve mode, every local MCP JSON-RPC request is also recorded in an +in-process call log with bounded previews. Clients can request the current log +with `get_mcp_calls` and receive live `mcp_call` events for subsequent calls, +including server, method/tool name, status, elapsed time, and truncated +argument/result previews. + ```json { "mcpServers": { --- a/src/jcode/core/agent.ss +++ b/src/jcode/core/agent.ss @@ -30,6 +30,7 @@ ./models :jcode/provider/provider :jcode/tool/registry + :jcode/mcp/client :jcode/guardrails/guardrails :jcode/guardrails/nudge :jcode/guardrails/respond @@ -1381,13 +1382,15 @@ Be concise. Prefer edit over write for modifying existing files. ;; commit 0a3a04d (TUI worker thread). (let ((err-port (current-error-port)) (log-lvl (current-log-level)) - (tool-cb (current-tool-cb))) + (tool-cb (current-tool-cb)) + (mcp-cb (current-mcp-call-cb))) (let ((threads (map (lambda (tc) (spawn (lambda () (parameterize ((current-error-port err-port) (current-log-level log-lvl) - (current-tool-cb tool-cb)) + (current-tool-cb tool-cb) + (current-mcp-call-cb mcp-cb)) (execute-single-tool tc))))) tool-calls))) (map thread-join! threads))))) --- a/src/jcode/mcp/client.ss +++ b/src/jcode/mcp/client.ss @@ -13,7 +13,9 @@ mcp-stop! mcp-stop-all! mcp-active-servers - mcp-server-pids) + mcp-server-pids + current-mcp-call-cb + mcp-call-history) (import :std/text/json :std/misc/string @@ -46,6 +48,66 @@ ;; round-trip, which could hang forever if a server is slow or dead. (def *mcp-tool-counts* (make-hash-table)) +;; Optional callback invoked after every MCP JSON-RPC request completes. Serve +;; mode binds this so connected clients can render a live MCP page. +(def current-mcp-call-cb (make-parameter #f)) + +;; Newest first. The serve protocol reverses this when replaying history. +(def *mcp-call-log* '()) +(def *mcp-call-log-lock* (raw-make-mutex)) + +(def (current-time-ms) + (let ((t (current-time))) + (+ (* (time-second t) 1000) + (quotient (time-nanosecond t) 1000000)))) + +(def (mcp-truncate s max-len) + (cond + ((not (string? s)) "") + ((<= (string-length s) max-len) s) + (else + (string-append + (substring s 0 max-len) + (format "\n... truncated ~a bytes ..." + (- (string-length s) max-len)))))) + +(def (mcp-preview value) + (mcp-truncate + (try (json-object->string value) + (catch (_) (format "~a" value))) + 4000)) + +(def (mcp-call-history) + (with-mutex *mcp-call-log-lock* + (reverse *mcp-call-log*))) + +(def (mcp-clear-call-history!) + (with-mutex *mcp-call-log-lock* + (set! *mcp-call-log* '()))) + +(def (record-mcp-call! conn method id params started-ms status result error-text) + (let* ((rec (make-hash-table)) + (duration (- (current-time-ms) started-ms)) + (tool-name (and (hash-table? params) (hash-get params "name" #f))) + (args (and (hash-table? params) (hash-get params "arguments" #f)))) + (hash-put! rec "server" (mcp-conn-name conn)) + (hash-put! rec "method" method) + (hash-put! rec "id" id) + (hash-put! rec "status" status) + (hash-put! rec "started_ms" started-ms) + (hash-put! rec "duration_ms" duration) + (hash-put! rec "params" (mcp-preview params)) + (when tool-name (hash-put! rec "tool" tool-name)) + (when args (hash-put! rec "arguments" (mcp-preview args))) + (when result (hash-put! rec "result" (mcp-preview result))) + (when error-text (hash-put! rec "error" (mcp-truncate error-text 4000))) + (with-mutex *mcp-call-log-lock* + (set! *mcp-call-log* (cons rec *mcp-call-log*))) + (let ((cb (current-mcp-call-cb))) + (when cb + (try (cb rec) (catch (_) (void))))) + rec)) + ;; --- subprocess management --- (def (mcp-start name command args) @@ -75,7 +137,8 @@ (def (mcp-stop-all!) (for-each mcp-stop! *mcp-servers*) (set! *mcp-servers* '()) - (set! *mcp-tool-counts* (make-hash-table))) + (set! *mcp-tool-counts* (make-hash-table)) + (mcp-clear-call-history!)) (def (mcp-server-pids) "Return a list of PIDs for all active MCP server subprocesses." @@ -120,7 +183,15 @@ (display json-str (mcp-conn-to-stdin conn)) (newline (mcp-conn-to-stdin conn)) (flush-output-port (mcp-conn-to-stdin conn)) - (mcp-read-response conn id))))) + (let ((started-ms (current-time-ms))) + (try + (let ((result (mcp-read-response conn id))) + (record-mcp-call! conn method id params started-ms "ok" result #f) + result) + (catch (e) + (record-mcp-call! conn method id params started-ms "error" #f + (err->string e)) + (raise e))))))))