feat(minecraft): expect guardrails and a working gaze api

Add toCoord/cloneVec3 helpers, return structured telemetry from goToPlayer/goToCoordinate with ok/startPos/endPos/movedDistance/distanceToTargetBefore/distanceToTargetAfter fields, track lastPlannerOutcome summary (actionCount/okCount/errorCount/returnValue/logs) in Brain and inject as [SCRIPT] context with truncation, implement expect/expectMoved/expectNear guardrail tools in JavaScriptPlanner to validate action

Author: shinohara-rin

services/minecraft/src/cognitive/action/llm-actions.ts

@@ -24,6 +24,14 @@ function formatWearingItem(slot: string, item: string | undefined): string {
   return item ? `\n${slot}: ${item}` : ''
 }
 
+function toCoord(pos: { x: number, y: number, z: number }) {
+  return { x: pos.x, y: pos.y, z: pos.z }
+}
+
+function cloneVec3(pos: { x: number, y: number, z: number }): Vec3 {
+  return new Vec3(pos.x, pos.y, pos.z)
+}
+
 export const actionsList: Action[] = [
   {
     name: 'chat',
@@ -147,9 +155,31 @@ export const actionsList: Action[] = [
       closeness: z.number().describe('How close to get to the player in blocks.').min(0),
     }),
     perform: mineflayer => async (player_name: string, closeness: number) => {
+      const getPlayerPos = () => {
+        const entity = mineflayer.bot.players[player_name]?.entity
+        return entity ? cloneVec3(entity.position) : null
+      }
+
+      const selfStart = cloneVec3(mineflayer.bot.entity.position)
+      const targetStart = getPlayerPos()
+      const distanceToTargetBefore = targetStart ? selfStart.distanceTo(targetStart) : null
+
       // TODO estimate time cost based on distance, trigger failure if time runs out
-      await skills.goToPlayer(mineflayer, player_name, closeness)
-      return `Arrived at player [${player_name}]`
+      const ok = await skills.goToPlayer(mineflayer, player_name, closeness)
+
+      const selfEnd = cloneVec3(mineflayer.bot.entity.position)
+      const targetEnd = getPlayerPos()
+      const distanceToTargetAfter = targetEnd ? selfEnd.distanceTo(targetEnd) : null
+
+      return {
+        ok,
+        target: { player_name, closeness },
+        startPos: toCoord(selfStart),
+        endPos: toCoord(selfEnd),
+        movedDistance: selfStart.distanceTo(selfEnd),
+        distanceToTargetBefore,
+        distanceToTargetAfter,
+      }
     },
   },
   {
@@ -197,8 +227,25 @@ export const actionsList: Action[] = [
       closeness: z.number().describe('0 If want to be exactly at the position, otherwise a positive number in blocks for leniency.').min(0),
     }),
     perform: mineflayer => async (x: number, y: number, z: number, closeness: number) => {
-      await skills.goToPosition(mineflayer, x, y, z, closeness)
-      return `Arrived at coordinate [${x}, ${y}, ${z}]`
+      const selfStart = cloneVec3(mineflayer.bot.entity.position)
+      const targetVec = new Vec3(x, y, z)
+      const distanceToTargetBefore = selfStart.distanceTo(targetVec)
+
+      const ok = await skills.goToPosition(mineflayer, x, y, z, closeness)
+
+      const selfEnd = cloneVec3(mineflayer.bot.entity.position)
+      const distanceToTargetAfter = selfEnd.distanceTo(targetVec)
+
+      return {
+        ok,
+        target: { x, y, z, closeness },
+        startPos: toCoord(selfStart),
+        endPos: toCoord(selfEnd),
+        movedDistance: selfStart.distanceTo(selfEnd),
+        distanceToTargetBefore,
+        distanceToTargetAfter,
+        withinCloseness: distanceToTargetAfter <= closeness,
+      }
     },
   },
   {

services/minecraft/src/cognitive/conscious/brain.ts

@@ -36,6 +36,19 @@ interface QueuedEvent {
   reject: (err: Error) => void
 }
 
+interface PlannerOutcomeSummary {
+  actionCount: number
+  okCount: number
+  errorCount: number
+  returnValue?: string
+  logs: string[]
+  updatedAt: number
+}
+
+function truncateForPrompt(value: string, maxLength = 220): string {
+  return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}...`
+}
+
 export class Brain {
   private debugService: DebugService
   private readonly planner = new JavaScriptPlanner()
@@ -49,6 +62,7 @@ export class Brain {
   private lastHumanChatAt = 0
   private botUsername = ''
   private lastContextView: string | undefined
+  private lastPlannerOutcome: PlannerOutcomeSummary | undefined
   private conversationHistory: Message[] = []
 
   constructor(private readonly deps: BrainDeps) {
@@ -285,6 +299,15 @@ export class Brain {
         },
       )
 
+      this.lastPlannerOutcome = {
+        actionCount: runResult.actions.length,
+        okCount: runResult.actions.filter(item => item.ok).length,
+        errorCount: runResult.actions.filter(item => !item.ok).length,
+        returnValue: runResult.returnValue,
+        logs: runResult.logs.slice(-3),
+        updatedAt: Date.now(),
+      }
+
       if (runResult.actions.length === 0 || runResult.actions.every(item => item.action.tool === 'skip')) {
         this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)')
         return
@@ -347,7 +370,16 @@ export class Brain {
       parts.push(`[STATE] giveUp active (${remainingSec}s left). reason=${this.giveUpReason ?? 'unknown'}`)
     }
 
-    parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, mem, lastRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.')
+    if (this.lastPlannerOutcome) {
+      const ageMs = Date.now() - this.lastPlannerOutcome.updatedAt
+      const returnValue = truncateForPrompt(this.lastPlannerOutcome.returnValue ?? 'undefined')
+      const logs = this.lastPlannerOutcome.logs.length > 0
+        ? this.lastPlannerOutcome.logs.map((line, index) => `#${index + 1} ${truncateForPrompt(line, 120)}`).join(' | ')
+        : '(none)'
+      parts.push(`[SCRIPT] Last eval ${ageMs}ms ago: return=${returnValue}; actions=${this.lastPlannerOutcome.actionCount} (ok=${this.lastPlannerOutcome.okCount}, err=${this.lastPlannerOutcome.errorCount}); logs=${logs}`)
+    }
+
+    parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, mem, lastRun, prevRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.')
 
     return parts.join('\n\n')
   }

services/minecraft/src/cognitive/conscious/js-planner.test.ts

@@ -112,4 +112,38 @@ describe('JavaScriptPlanner', () => {
 
     await expect(planner.evaluate('while (true) {}', actions, globals, executeAction)).rejects.toThrow(/Script execution timed out/i)
   })
+
+  it('supports expectation guardrails on structured action telemetry', async () => {
+    const planner = new JavaScriptPlanner()
+    const executeAction = vi.fn(async () => ({
+      ok: true,
+      movedDistance: 1.25,
+      distanceToTargetAfter: 1.5,
+      endPos: { x: 8, y: 64, z: 4 },
+    }))
+
+    const planned = await planner.evaluate(`
+      const nav = await goToPlayer({ player_name: "Alex", closeness: 2 })
+      expect(nav.ok, "go failed")
+      expectMoved(1)
+      expectNear(2)
+      expectNear({ x: 7, y: 64, z: 4 }, 2)
+    `, actions, globals, executeAction)
+
+    expect(planned.actions).toHaveLength(1)
+    expect(planned.actions[0]?.ok).toBe(true)
+  })
+
+  it('throws when expectation guardrail fails', async () => {
+    const planner = new JavaScriptPlanner()
+    const executeAction = vi.fn(async () => ({
+      ok: true,
+      movedDistance: 0.1,
+    }))
+
+    await expect(planner.evaluate(`
+      await goToPlayer({ player_name: "Alex", closeness: 2 })
+      expectMoved(1, "did not move enough")
+    `, actions, globals, executeAction)).rejects.toThrow(/Expectation failed: did not move enough/i)
+  })
 })

services/minecraft/src/cognitive/conscious/js-planner.ts

@@ -35,6 +35,13 @@ function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null && !Array.isArray(value)
 }
 
+function isCoord(value: unknown): value is { x: number, y: number, z: number } {
+  return isRecord(value)
+    && typeof value.x === 'number'
+    && typeof value.y === 'number'
+    && typeof value.z === 'number'
+}
+
 function deepFreeze<T>(value: T): T {
   if (!value || typeof value !== 'object')
     return value
@@ -114,6 +121,10 @@ export class JavaScriptPlanner {
         ? undefined
         : inspect(result, { depth: 2, breakLength: 100 })
 
+      if (isRecord(this.sandbox.lastRun)) {
+        this.sandbox.lastRun.returnValue = returnValue
+      }
+
       return {
         actions: run.executed,
         logs: run.logs,
@@ -143,9 +154,93 @@ export class JavaScriptPlanner {
       this.activeRun.logs.push(rendered)
       return rendered
     })
+    this.defineGlobalTool('expect', (condition: unknown, message?: unknown) => {
+      if (condition)
+        return true
+
+      const detail = typeof message === 'string' && message.trim().length > 0
+        ? message
+        : 'Condition evaluated to false'
+      throw new Error(`Expectation failed: ${detail}`)
+    })
+    this.defineGlobalTool('expectMoved', (minBlocks?: unknown, message?: unknown) => {
+      const threshold = typeof minBlocks === 'number' ? minBlocks : 0.5
+      const telemetry = this.getLastActionResultRecord()
+      const movedDistance = typeof telemetry?.movedDistance === 'number'
+        ? telemetry.movedDistance
+        : null
+
+      if (movedDistance === null) {
+        throw new Error('Expectation failed: expectMoved() requires last action result with movedDistance telemetry')
+      }
+
+      if (movedDistance >= threshold)
+        return true
+
+      const detail = typeof message === 'string' && message.trim().length > 0
+        ? message
+        : `Expected movedDistance >= ${threshold}, got ${movedDistance}`
+      throw new Error(`Expectation failed: ${detail}`)
+    })
+    this.defineGlobalTool('expectNear', (targetOrMaxDist?: unknown, maxDistOrMessage?: unknown, maybeMessage?: unknown) => {
+      const telemetry = this.getLastActionResultRecord()
+
+      let target: { x: number, y: number, z: number } | null = null
+      let maxDist = 2
+      let message: string | undefined
+
+      if (isCoord(targetOrMaxDist)) {
+        target = { x: targetOrMaxDist.x, y: targetOrMaxDist.y, z: targetOrMaxDist.z }
+        if (typeof maxDistOrMessage === 'number')
+          maxDist = maxDistOrMessage
+        if (typeof maybeMessage === 'string')
+          message = maybeMessage
+      }
+      else {
+        if (typeof targetOrMaxDist === 'number')
+          maxDist = targetOrMaxDist
+        if (typeof maxDistOrMessage === 'string')
+          message = maxDistOrMessage
+      }
+
+      let distance: number | null = null
+      if (target) {
+        const endPos = isCoord(telemetry?.endPos) ? telemetry.endPos : null
+        if (!endPos) {
+          throw new Error('Expectation failed: expectNear(target) requires last action result with endPos telemetry')
+        }
+
+        const dx = endPos.x - target.x
+        const dy = endPos.y - target.y
+        const dz = endPos.z - target.z
+        distance = Math.sqrt(dx * dx + dy * dy + dz * dz)
+      }
+      else if (typeof telemetry?.distanceToTargetAfter === 'number') {
+        distance = telemetry.distanceToTargetAfter
+      }
+
+      if (distance === null) {
+        throw new Error('Expectation failed: expectNear() requires target argument or last action distanceToTargetAfter telemetry')
+      }
+
+      if (distance <= maxDist)
+        return true
+
+      const detail = message ?? `Expected distance <= ${maxDist}, got ${distance}`
+      throw new Error(`Expectation failed: ${detail}`)
+    })
     this.defineGlobalValue('mem', {})
   }
 
+  private getLastActionResultRecord(): Record<string, unknown> | null {
+    const lastAction = this.sandbox.lastAction
+    if (!isRecord(lastAction))
+      return null
+
+    const result = lastAction.result
+    return isRecord(result) ? result : null
+  }
+
   private installActionTools(availableActions: Action[]): void {
     for (const action of availableActions) {
       this.defineGlobalTool(action.name, async (...args: unknown[]) => {

services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts

@@ -108,6 +108,7 @@ You are an autonomous agent playing Minecraft.
    - Use \`await\` on tool calls when later logic depends on the result.
    - Globals refreshed every turn: \`snapshot\`, \`self\`, \`environment\`, \`social\`, \`threat\`, \`attention\`, \`autonomy\`, \`event\`, \`now\`.
    - Persistent globals: \`mem\` (cross-turn memory), \`lastRun\` (this run), \`prevRun\` (previous run), \`lastAction\` (latest action result), \`log(...)\`.
+   - Last script outcome is also echoed in the next turn as \`[SCRIPT]\` context (return value, action stats, and logs).
    - Maximum actions per turn: 5.
 
 # Environment & Global Semantics
@@ -142,13 +143,37 @@ Call tool functions directly.
 Use \`await\` when branching on action outcomes.
 If you want to do nothing, call \`await skip()\`.
 You can also use \`use(toolName, paramsObject)\` for dynamic tool calls.
+Use built-in guardrails to verify outcomes: \`expect(...)\`, \`expectMoved(...)\`, \`expectNear(...)\`.
 
 Examples:
 - \`await chat("hello")\`
 - \`const sent = await chat("HP=" + self.health); log(sent)\`
 - \`const arrived = await goToPlayer({ player_name: "Alex", closeness: 2 }); if (!arrived) await chat("failed")\`
 - \`if (self.health < 10) await consume({ item_name: "bread" })\`
 - \`await skip()\`
+- \`const nav = await goToCoordinate({ x: 12, y: 64, z: -5, closeness: 2 }); expect(nav.ok, "navigation failed"); expectMoved(0.8); expectNear(2.5)\`
+
+Guardrail semantics:
+- \`expect(condition, message?)\`: throw if condition is falsy.
+- \`expectMoved(minBlocks = 0.5, message?)\`: checks last action telemetry \`movedDistance\`.
+- \`expectNear(targetOrMaxDist = 2, maxDist?, message?)\`:
+  - \`expectNear(2.5)\` uses last action telemetry \`distanceToTargetAfter\`.
+  - \`expectNear({ x, y, z }, 2)\` uses last action telemetry \`endPos\`.
+
+Common patterns:
+- Follow + detach for exploration:
+  - \`await followPlayer({ player_name: "laggy_magpie", follow_dist: 2 })\`
+  - \`const nav = await goToCoordinate({ x: 120, y: 70, z: -30, closeness: 2 }) // detaches follow automatically\`
+  - \`expect(nav.ok, "failed to reach exploration point")\`
+- Confirm movement before claiming progress:
+  - \`const r = await goToPlayer({ player_name: "Alex", closeness: 2 })\`
+  - \`expect(r.ok, "goToPlayer failed")\`
+  - \`expectMoved(1, "I did not actually move")\`
+  - \`expectNear(3, "still too far from player")\`
+- Gaze as weak hint only:
+  - \`const gaze = environment.nearbyPlayersGaze.find(g => g.name === "Alex")\`
+  - \`if (event.type === "perception" && event.payload?.type === "chat_message" && gaze?.hitBlock)\`
+  - \`  await goToCoordinate({ x: gaze.hitBlock.pos.x, y: gaze.hitBlock.pos.y, z: gaze.hitBlock.pos.z, closeness: 2 })\`
 
 # Usage Convention (Important)
 - Plan with \`mem.plan\`, execute in small steps, and verify each step before continuing.

services/minecraft/src/cognitive/reflex/gaze.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest'
+import { Vec3 } from 'vec3'
+
+import { rayTraceBlockFromEntity } from './gaze'
+
+describe('rayTraceBlockFromEntity', () => {
+  it('detects block when player looks downward', () => {
+    const bot = {
+      blockAt(pos: Vec3) {
+        if (pos.x === 0 && pos.y === 63 && pos.z === -3) {
+          return {
+            name: 'grass_block',
+            position: new Vec3(0, 63, -3),
+          }
+        }
+        return { name: 'air', position: pos }
+      },
+    } as any
+
+    const entity = {
+      type: 'player',
+      username: 'tester',
+      position: new Vec3(0, 64, 0),
+      yaw: 0,
+      pitch: Math.PI / 4,
+    }
+
+    const result = rayTraceBlockFromEntity(bot, entity, { maxDistance: 8, step: 0.1 })
+    expect(result.hitBlock?.name).toBe('grass_block')
+    expect(result.hitBlock?.pos).toEqual({ x: 0, y: 63, z: -3 })
+  })
+
+  it('returns null hitBlock when no solid block is intersected', () => {
+    const bot = {
+      blockAt(pos: Vec3) {
+        return { name: 'air', position: pos }
+      },
+    } as any
+
+    const entity = {
+      type: 'player',
+      username: 'tester',
+      position: new Vec3(0, 64, 0),
+      yaw: 0,
+      pitch: -Math.PI / 4,
+    }
+
+    const result = rayTraceBlockFromEntity(bot, entity, { maxDistance: 8, step: 0.1 })
+    expect(result.hitBlock).toBeNull()
+  })
+})