Merge pull request #2546 from dgageot/board/llm-as-judge-for-auto-approving-tool-1563505f

feat(hooks): add 'type: model' hook and integrate pre_tool_use into approval flow

Author: dgageot

agent-schema.json

@@ -701,15 +701,16 @@
         },
         "type": {
           "type": "string",
-          "description": "Type of hook. 'command' executes a shell command; 'builtin' invokes a named in-process Go function registered by the runtime. The docker-agent runtime ships these builtins: 'add_date' (turn_start: today's date), 'add_environment_info' (session_start: cwd, git, OS, arch), 'add_prompt_files' (turn_start: contents of named files looked up in the workdir hierarchy and the home directory), 'add_git_status' (turn_start: `git status --short --branch`), 'add_git_diff' (turn_start: `git diff --stat`, or full diff with args=['full']), 'add_directory_listing' (session_start: top-level entries of cwd), 'add_user_info' (session_start: current OS user and hostname), 'add_recent_commits' (session_start: `git log --oneline -n N`, default N=10, override via args=['<N>']), 'max_iterations' (before_llm_call: hard stop after N model calls; args=['<N>'] required).",
+          "description": "Type of hook. 'command' executes a shell command; 'builtin' invokes a named in-process Go function registered by the runtime; 'model' asks an LLM and translates its reply into the hook's native output (used for LLM-as-a-judge pre_tool_use, summarizers, etc., with no Go code). The docker-agent runtime ships these builtins: 'add_date' (turn_start: today's date), 'add_environment_info' (session_start: cwd, git, OS, arch), 'add_prompt_files' (turn_start: contents of named files looked up in the workdir hierarchy and the home directory), 'add_git_status' (turn_start: `git status --short --branch`), 'add_git_diff' (turn_start: `git diff --stat`, or full diff with args=['full']), 'add_directory_listing' (session_start: top-level entries of cwd), 'add_user_info' (session_start: current OS user and hostname), 'add_recent_commits' (session_start: `git log --oneline -n N`, default N=10, override via args=['<N>']), 'max_iterations' (before_llm_call: hard stop after N model calls; args=['<N>'] required).",
           "enum": [
             "command",
-            "builtin"
+            "builtin",
+            "model"
           ]
         },
         "command": {
           "type": "string",
-          "description": "Shell command (type=command) or builtin name (type=builtin) to invoke. Command hooks receive JSON input via stdin with tool/session information."
+          "description": "Shell command (type=command) or builtin name (type=builtin) to invoke. Command hooks receive JSON input via stdin with tool/session information. Ignored when type=model."
         },
         "args": {
           "type": "array",
@@ -744,13 +745,38 @@
             "block"
           ],
           "default": "warn"
+        },
+        "model": {
+          "type": "string",
+          "description": "Model spec ('provider/model', e.g. 'openai/gpt-4o-mini') invoked by type=model hooks. Required for that type, ignored otherwise."
+        },
+        "prompt": {
+          "type": "string",
+          "description": "User-message template rendered for each invocation of a type=model hook. Parsed as a Go text/template with the hook Input as the data context: {{ .ToolName }}, {{ .ToolInput }}, {{ .StopResponse }}, etc. Required for type=model."
+        },
+        "schema": {
+          "type": "string",
+          "description": "Well-known response interpretation for type=model hooks. Empty: return the model's reply as additional_context. 'pre_tool_use_decision': ask the model for {decision, reason} JSON and produce a permission_decision verdict (allow|ask|deny)."
         }
       },
       "required": [
-        "type",
-        "command"
+        "type"
       ],
-      "additionalProperties": false
+      "additionalProperties": false,
+      "allOf": [
+        {
+          "if": {"properties": {"type": {"const": "command"}}, "required": ["type"]},
+          "then": {"required": ["command"]}
+        },
+        {
+          "if": {"properties": {"type": {"const": "builtin"}}, "required": ["type"]},
+          "then": {"required": ["command"]}
+        },
+        {
+          "if": {"properties": {"type": {"const": "model"}}, "required": ["type"]},
+          "then": {"required": ["model", "prompt"]}
+        }
+      ]
     },
     "ModelConfig": {
       "type": "object",

docs/configuration/hooks/index.md

@@ -471,6 +471,66 @@ hooks:
 
 Return nothing to fall through to the usual interactive confirmation.
 
+### LLM as a Judge (Auto-Approving Tool Calls)
+
+The `model` hook type asks an LLM and translates its reply into the
+hook's native output — no Go code, no shell glue, no JSON parsing on
+your side. Combined with the well-known `pre_tool_use_decision`
+schema it gives you a fully-configurable LLM judge that decides
+`allow` / `ask` / `deny` per tool call.
+
+```yaml
+hooks:
+  pre_tool_use:
+    - matcher: "shell|edit_file|mcp:.*"
+      hooks:
+        - type: model
+          model: openai/gpt-4o-mini
+          timeout: 15
+          schema: pre_tool_use_decision
+          prompt: |
+            You are a security judge for an autonomous agent.
+            Decide whether this tool call is safe to auto-approve.
+
+            Tool: {{ .ToolName }}
+            Args: {{ .ToolInput | toJSON }}
+
+            Project rules:
+            - Reads under the working directory are safe.
+            - Writes to ~/.ssh / ~/.aws / ~/.docker are deny.
+```
+
+| Field    | Required          | Description                                                                                                                                                                |
+| -------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `model`  | yes               | Model spec (`provider/model`, e.g. `openai/gpt-4o-mini`). The judge model — small/cheap is recommended.                                                                    |
+| `prompt` | yes               | Go [`text/template`](https://pkg.go.dev/text/template) body. Sees the hook [Input](#hook-input) as data, plus the `toJSON` and `truncate <n>` helpers.                     |
+| `schema` | no                | Well-known response interpretation. `pre_tool_use_decision` produces a `permission_decision` verdict; omit for free-form text injected as `additional_context`.            |
+| `timeout`| no (default 60s)  | Per-call timeout. **Timeouts fail closed (deny) for `pre_tool_use`** regardless of any other setting. Match it to your judge model's typical latency plus a small buffer. |
+
+The `pre_tool_use_decision` schema constrains the judge to reply with
+strict `{decision, reason}` JSON. Providers that honor structured
+output (OpenAI, ...) are asked to emit that shape directly; on
+providers that ignore it the framework still parses tolerant
+JSON-in-text. Anything unparseable propagates as a hook error and the
+executor falls closed (deny) on `pre_tool_use`.
+
+Pair it with deterministic `permissions:` rules so destructive calls
+(e.g. `sudo`, `rm -rf`) are blocked even if the judge is misled, and
+obvious read-only calls bypass the LLM entirely. See
+[`examples/llm_judge.yaml`](https://github.com/docker/docker-agent/blob/main/examples/llm_judge.yaml)
+for a complete configuration.
+
+**Security considerations**:
+
+- **Sensitive data**: Tool arguments (including file paths, command
+  arguments, and any other parameters) are sent to the judge LLM. Avoid
+  using the judge on tools that handle secrets, or ensure your judge
+  model is self-hosted.
+- **Defense in depth**: The judge should not be your only security
+  layer. Use deterministic `permissions:` rules to block obviously
+  dangerous operations (e.g., `sudo`, `rm -rf`) before the judge sees
+  them, as shown in the example configuration.
+
 </div>
 
 ## CLI Flags

examples/llm_judge.yaml

@@ -0,0 +1,90 @@
+#!/usr/bin/env docker agent run
+#
+# LLM-as-a-judge — auto-approve tool calls with a small model
+# ==========================================================================
+#
+# This example wires three layers of tool-call control:
+#
+#   1. Deterministic permissions  — `permissions:` block. Hard deny rules
+#      catch obvious abuse (sudo, rm -rf, etc) regardless of what the
+#      judge says, and obvious read-only ops are allow-listed so the
+#      judge isn't paid for every harmless call.
+#
+#   2. LLM judge (this file's centerpiece) — a `pre_tool_use` hook of
+#      type `model` that asks a small LLM to verdict every shell / edit
+#      / MCP call that fell through the deterministic rules. The
+#      `pre_tool_use_decision` schema constrains the model to reply
+#      with strict {decision, reason} JSON; the framework translates
+#      that into the runtime's permission_decision pipeline.
+#
+#   3. User confirmation — anything the judge said `ask` on falls
+#      through to the normal TUI prompt. So the human is still in the
+#      loop on ambiguous calls.
+#
+# This config uses ZERO Go code: the `type: model` hook is provided by
+# the framework, no builtin to register, no shell glue, no jq+curl.
+#
+
+agents:
+  root:
+    model: openai/gpt-4o
+    description: Agent supervised by an LLM judge for tool calls.
+    instruction: |
+      You are a helpful assistant with access to shell and filesystem tools.
+      Use them to help the user. The user's machine is supervised by an
+      LLM judge that decides which of your tool calls are auto-approved.
+
+    toolsets:
+      - type: shell
+      - type: filesystem
+
+    hooks:
+      # Layer 2: LLM judge. Scope it via the matcher so we don't pay
+      # judge latency on cheap tools.
+      pre_tool_use:
+        - matcher: "shell|edit_file|mcp:.*"
+          hooks:
+            - type: model
+              model: openai/gpt-4o-mini
+              # 15s is plenty for a small model; the judge fails closed
+              # (deny) on timeout. Pair with permissions.allow for
+              # read-only ops to keep latency budget under control.
+              timeout: 15
+              schema: pre_tool_use_decision
+              prompt: |
+                You are a security judge for an autonomous agent.
+                Decide whether this tool call is safe to auto-approve.
+
+                Tool: {{ .ToolName }}
+                Args: {{ .ToolInput | toJSON }}
+
+                Project rules:
+                - Reads under the working directory are safe.
+                - Network egress to anything other than github.com or
+                  pypi.org should be `ask`.
+                - Any write under ~/.ssh, ~/.aws, or ~/.docker is `deny`.
+                - When in doubt, prefer `ask` (the user is then asked).
+
+      # Layer 3 (audit): log every verdict so the policy can be tuned.
+      post_tool_use:
+        - matcher: "*"
+          hooks:
+            - type: command
+              timeout: 5
+              command: |
+                INPUT=$(cat)
+                TS=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+                TOOL=$(echo "$INPUT" | jq -r '.tool_name')
+                echo "[$TS] $TOOL completed" >> /tmp/agent-judge-audit.log
+
+# Layer 1: deterministic rules. Evaluated BEFORE any hook fires, so
+# they short-circuit the judge entirely for obvious cases.
+permissions:
+  deny:
+    - "shell:cmd=*sudo*"
+    - "shell:cmd=*rm -rf*"
+    - "shell:cmd=*mkfs*"
+    - "shell:cmd=*dd if=*"
+    - "edit_file:path=/etc/*"
+  allow:
+    - "read_*"

pkg/config/latest/types.go

@@ -1816,6 +1816,10 @@ type HookDefinition struct {
 	//                 is owned by the runtime; the docker-agent runtime
 	//                 ships add_date, add_environment_info, and
 	//                 add_prompt_files.
+	//   - "model":    ask an LLM and translate its reply into the hook's
+	//                 native output. See Model / Prompt / Schema. Used to
+	//                 implement "LLM as a judge" pre_tool_use hooks,
+	//                 turn-start summarizers, etc., with no Go code.
 	Type string `json:"type" yaml:"type"`
 
 	// Command is the shell command (Type==command) or the builtin name
@@ -1839,6 +1843,25 @@ type HookDefinition struct {
 
 	// OnError controls non-fail-closed hook failures: warn (default), ignore, or block.
 	OnError string `json:"on_error,omitempty" yaml:"on_error,omitempty"`
+
+	// Model is the model spec ("provider/model", e.g. "openai/gpt-4o-mini")
+	// invoked by Type==model hooks. Required for that type, ignored
+	// otherwise.
+	Model string `json:"model,omitempty" yaml:"model,omitempty"`
+
+	// Prompt is the user-message template rendered for each invocation
+	// of a Type==model hook. It is parsed as a Go text/template with the
+	// hook [Input] as the data context (so {{ .ToolName }},
+	// {{ .ToolInput }}, etc. work). Required for Type==model.
+	Prompt string `json:"prompt,omitempty" yaml:"prompt,omitempty"`
+
+	// Schema selects a well-known response interpretation for Type==model
+	// hooks. The empty value means "return the model's reply as
+	// additional_context". Other values (registered by the runtime) ask
+	// the provider for strict-JSON output and translate the result into
+	// the right Output shape (e.g. "pre_tool_use_decision" produces a
+	// permission_decision verdict).
+	Schema string `json:"schema,omitempty" yaml:"schema,omitempty"`
 }
 
 // GetTimeout returns the per-hook execution timeout, defaulting to 60
@@ -2044,8 +2067,15 @@ func (h *HookDefinition) validate(prefix string, index int) error {
 		if h.Command == "" {
 			return fmt.Errorf("hooks.%s[%d]: command must name the builtin to invoke", prefix, index)
 		}
+	case "model":
+		if h.Model == "" {
+			return fmt.Errorf("hooks.%s[%d]: model is required for model hooks (e.g. 'openai/gpt-4o-mini')", prefix, index)
+		}
+		if h.Prompt == "" {
+			return fmt.Errorf("hooks.%s[%d]: prompt is required for model hooks", prefix, index)
+		}
 	default:
-		return fmt.Errorf("hooks.%s[%d]: unsupported hook type '%s' (supported: 'command', 'builtin')", prefix, index, h.Type)
+		return fmt.Errorf("hooks.%s[%d]: unsupported hook type '%s' (supported: 'command', 'builtin', 'model')", prefix, index, h.Type)
 	}
 
 	return nil

pkg/hooks/aggregate_test.go

@@ -0,0 +1,106 @@
+package hooks
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestAggregateTracksMostRestrictiveDecision pins the new
+// Result.Decision contract: when multiple pre_tool_use hooks fire on a
+// single tool call, the aggregated verdict is the most-restrictive
+// (Deny > Ask > Allow). The runtime's tool-approval flow consults this
+// to short-circuit the user prompt for Allow and to escalate Ask, so
+// the ordering must be stable.
+func TestAggregateTracksMostRestrictiveDecision(t *testing.T) {
+	t.Parallel()
+
+	mk := func(d Decision, reason string) hookResult {
+		return hookResult{HandlerResult: HandlerResult{Output: &Output{
+			HookSpecificOutput: &HookSpecificOutput{
+				HookEventName:            EventPreToolUse,
+				PermissionDecision:       d,
+				PermissionDecisionReason: reason,
+			},
+		}}}
+	}
+
+	cases := []struct {
+		name        string
+		results     []hookResult
+		wantVerdict Decision
+		wantReason  string
+		wantAllowed bool
+	}{
+		{
+			name:        "no decision: Allowed=true, Decision empty",
+			results:     []hookResult{{}},
+			wantVerdict: "",
+			wantAllowed: true,
+		},
+		{
+			name:        "single allow",
+			results:     []hookResult{mk(DecisionAllow, "safe")},
+			wantVerdict: DecisionAllow,
+			wantReason:  "safe",
+			wantAllowed: true,
+		},
+		{
+			name:        "single ask escalates over no decision",
+			results:     []hookResult{{}, mk(DecisionAsk, "unclear")},
+			wantVerdict: DecisionAsk,
+			wantReason:  "unclear",
+			wantAllowed: true, // Ask doesn't flip Allowed; the runtime handles the prompt.
+		},
+		{
+			name: "deny beats ask beats allow",
+			results: []hookResult{
+				mk(DecisionAllow, "looks fine"),
+				mk(DecisionAsk, "second-guess"),
+				mk(DecisionDeny, "destructive"),
+			},
+			wantVerdict: DecisionDeny,
+			wantReason:  "destructive",
+			wantAllowed: false,
+		},
+		{
+			name: "first reason wins on ties",
+			results: []hookResult{
+				mk(DecisionAsk, "first ask"),
+				mk(DecisionAsk, "second ask"),
+			},
+			wantVerdict: DecisionAsk,
+			wantReason:  "first ask",
+			wantAllowed: true,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			final := aggregate(tc.results, EventPreToolUse)
+			assert.Equal(t, tc.wantVerdict, final.Decision)
+			assert.Equal(t, tc.wantReason, final.DecisionReason)
+			assert.Equal(t, tc.wantAllowed, final.Allowed)
+		})
+	}
+}
+
+// TestAggregateDecisionEmptyForNonPreToolUse documents that
+// Result.Decision is meaningful only for pre_tool_use events. Other
+// events (turn_start, post_tool_use, ...) MUST leave it empty so a
+// runtime that consults it can't accidentally act on a stale verdict
+// from an unrelated hook.
+func TestAggregateDecisionEmptyForNonPreToolUse(t *testing.T) {
+	t.Parallel()
+
+	results := []hookResult{{HandlerResult: HandlerResult{Output: &Output{
+		HookSpecificOutput: &HookSpecificOutput{
+			HookEventName:      EventTurnStart,
+			PermissionDecision: DecisionAllow, // misconfigured but possible
+		},
+	}}}}
+
+	final := aggregate(results, EventTurnStart)
+	assert.Equal(t, Decision(""), final.Decision)
+	assert.Empty(t, final.DecisionReason)
+}

pkg/hooks/builtins/builtins.go

@@ -23,6 +23,10 @@
 // stable for its duration. max_iterations is stateful: its
 // per-session counter lives on the [State] returned by [Register];
 // the runtime clears it via [State.ClearSession] from session_end.
+//
+// LLM-as-a-judge hooks are NOT shipped here: write `type: model` with
+// `schema: pre_tool_use_decision` instead — see
+// pkg/hooks/shape_pre_tool_use_decision.go and examples/llm_judge.yaml.
 package builtins
 
 import (