feat(minecraft): use native tool calling for instant tasks

Author: shinohara-rin

services/minecraft/src/agents/action/adapter.ts

@@ -13,6 +13,25 @@ import { useLogger } from '../../utils/logger'
 import { generateActionSystemPrompt } from './system-prompt'
 import { actionsList } from './tools'
 
+/**
+ * Generate actionable suggestions for common error codes.
+ * These help the LLM understand what it can do to recover from failures.
+ */
+function getSuggestionForError(code: string): string {
+  switch (code) {
+    case 'RESOURCE_MISSING':
+      return 'Suggestion: Check your inventory first, then gather the missing resources or ask the player for help if you cannot find them.'
+    case 'TARGET_NOT_FOUND':
+      return 'Suggestion: The target could not be found. Ask the player for clarification or use exploration tools to locate it.'
+    case 'NO_PATH':
+      return 'Suggestion: Unable to reach the destination. Try finding an alternative route or ask the player for guidance.'
+    case 'TIMEOUT':
+      return 'Suggestion: The action timed out. Consider breaking it into smaller steps or trying again later.'
+    default:
+      return 'Suggestion: Review the error details and try a different approach, or ask the player for help.'
+  }
+}
+
 export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise<Agent> {
   const logger = useLogger()
   logger.log('Initializing action agent')
@@ -34,7 +53,9 @@ export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise<Ag
           // This allows the LLM to learn from tool failures during its reasoning phase
           if (error instanceof ActionError) {
             logger.withError(error).warn('Action failed during tool call')
-            return `[FAILED] ${error.code}: ${error.message}${error.context ? ` (${JSON.stringify(error.context)})` : ''}`
+            const contextStr = error.context ? `\nContext: ${JSON.stringify(error.context)}` : ''
+            const suggestion = getSuggestionForError(error.code)
+            return `[FAILED] ${error.code}: ${error.message}${contextStr}\n${suggestion}`
           }
           // Re-throw non-ActionError errors (unexpected failures)
           throw error

services/minecraft/src/agents/action/tools.ts

@@ -26,8 +26,19 @@ function formatWearingItem(slot: string, item: string | undefined): string {
 }
 
 export const actionsList: Action[] = [
-  // {
-  //   name: 'setReflexMode',
+  {
+    name: 'chat',
+    description: 'Send a chat message to players in the game. Use this to communicate, respond to questions, or announce what you are doing.',
+    execution: 'parallel',
+    schema: z.object({
+      message: z.string().describe('The message to send in chat.'),
+    }),
+    perform: mineflayer => (message: string): string => {
+      mineflayer.bot.chat(message)
+      return `Sent message: "${message}"`
+    },
+  },
+  // {\n  //   name: 'setReflexMode',
   //   description: 'Set (or clear) your reflex mode override. Use work/wander to disable idle-only reflex behaviors. Set override to null to return to automatic mode selection.',
   //   execution: 'sequential',
   //   schema: z.object({

services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts

@@ -17,15 +17,22 @@ export function generateBrainSystemPrompt(
     return now - timestamp <= windowMs
   }
 
-  const actionDefinitions = availableActions.map((a) => {
-    return {
-      name: a.name,
-      description: a.description,
-      execution: a.execution,
-    }
-  })
+  // Separate tools by execution type
+  const instantTools = availableActions.filter(a => a.execution === 'parallel')
+  const asyncActions = availableActions.filter(a => a.execution === 'sequential')
 
-  const availableActionsJson = JSON.stringify(actionDefinitions, null, 2)
+  const instantToolDefs = instantTools.map(a => ({
+    name: a.name,
+    description: a.description,
+  }))
+
+  const asyncActionDefs = asyncActions.map(a => ({
+    name: a.name,
+    description: a.description,
+  }))
+
+  const instantToolsJson = JSON.stringify(instantToolDefs, null, 2)
+  const asyncActionsJson = JSON.stringify(asyncActionDefs, null, 2)
 
   const recentWindowMs = 30_000
 
@@ -46,30 +53,46 @@ export function generateBrainSystemPrompt(
 You are playing the game Minecraft. Your user name is [${blackboard.selfUsername}].
 You are NOT an assistant. You are an autonomous agent with a virtual body, senses, and needs.
 
-# How you interact
-You cannot physically move your avatar directly. You interact with the world by emitting **Actions**.
-For actions you executed, you will be notified when they complete.
-You have no access to history events from previous turns yet. To remember things, you rely on the blackboard provided to you.
-
-
-Available Actions:
-${availableActionsJson}
-
-Rules:
-1. You can execute sequential actions, parallel actions, or chat actions
-1.1. Actions are processed in the order you output them.
-1.2. Sequential actions are awaited strictly in order.
-1.3. Chat actions are also awaited in order.
-1.4. Parallel actions are started immediately and run concurrently with later actions.
-1.5. If you need to claim an action succeeded, place that chat action AFTER the action that requires feedback.
-1.6. You do not have to react on partial feedback. If you are waiting for results, you can update thoughts and output an empty action list.
-2. The output must be valid JSON following the schema below
-3. Specify if a feedback is required for the action, i.e. whether you need to know the execution result for a good reason
-4. Failed actions will always result in a feedback
-5. Chat actions usually don't need feedbacks, because you can expect them to complete instantly and is unlikely to fail
-6. Often times you don't need to perform any action, in that case just use empty array for actions
-
-Output format:
+---
+
+# Instant Tools (Native Tool Calls)
+
+These tools execute IMMEDIATELY and return results within this same turn.
+Use them to gather information BEFORE deciding what actions to take.
+
+**How to use**: Invoke these by making native tool calls.
+**Important**: DO NOT put instant tools in the JSON "actions" array - they are separate!
+**On failure**: You will receive a [FAILED] message with suggestions. Use this to adjust your approach.
+
+${instantToolsJson}
+
+---
+
+# Async Actions (JSON Output)
+
+These actions take TIME to complete (movement, crafting, combat, etc.).
+They are queued and executed asynchronously after your response.
+
+**How to use**: Output these in the JSON "actions" array in your response.
+**Feedback**: You will receive feedback when they complete or fail.
+
+${asyncActionsJson}
+
+---
+
+# Response Format
+
+Your entire response must be valid JSON. Include only your thoughts, blackboard updates, and async actions.
+
+Rules for the "actions" array:
+1. Actions are processed in the order you output them
+2. Sequential actions are awaited strictly in order
+3. Parallel actions run concurrently with later actions
+4. Set "require_feedback": true if you need to know the result
+5. Failed actions always trigger feedback
+6. Use empty array if no action is needed
+
+Schema:
 {
   "thought": "Your current thought, internal monologue and memory. Put everything that might be useful for the next turn here",
   "blackboard": {
@@ -78,9 +101,8 @@ Output format:
     "executionStrategy": "Short-term plan if any."
   },
   "actions": [
-    {"type":"sequential","step":{"tool":"chat","params":{"message":"..."}},"require_feedback": false},
-    {"type":"parallel","step":{"tool":"action name","params":{...}},"require_feedback": true},
-    {"type":"sequential","step":{"tool":"action name","params":{...}},"require_feedback": false}
+    {"type":"sequential","step":{"tool":"goToPlayer","params":{"player_name":"Steve","closeness":3}},"require_feedback": true},
+    {"type":"parallel","step":{"tool":"collectBlocks","params":{"type":"oak_log","num":5}},"require_feedback": false}
   ]
 }