feat(minecraft): semantic percetion, belief and state pulling

Author: shinohara-rin

services/minecraft/src/cognitive/conscious/brain.ts

@@ -56,11 +56,16 @@ export class Brain {
   public init(bot: MineflayerWithAgents): void {
     this.log('INFO', 'Brain: Initializing...')
 
-    // Unified Perception Signal Handler
-    // this.deps.eventManager.on<PerceptionSignal>('perception', async (event) => {
-    //   this.log('INFO', `Brain: Received perception signal: ${event.payload.type} - ${event.payload.description}`)
-    //   await this.enqueueEvent(bot, event)
-    // })
+    // Perception Signal Handler - Only process chat messages for now
+    this.deps.eventManager.on<PerceptionSignal>('perception', async (event) => {
+      const signal = event.payload
+      // Only handle chat messages in the deliberative layer
+      if (signal.type !== 'chat_message')
+        return
+
+      this.log('INFO', `Brain: Received chat: ${signal.description}`)
+      await this.enqueueEvent(bot, event)
+    })
 
     // Listen to Task Execution Events (Action Feedback)
     this.deps.taskExecutor.on('action:completed', async ({ action, result }) => {

services/minecraft/src/cognitive/container.ts

@@ -117,8 +117,12 @@ export function createAgentContainer(options: {
       }),
 
     // Reflex Manager (Reactive Layer)
-    reflexManager: asFunction(({ eventBus, logger }) =>
-      new ReflexManager({ eventBus, logger }),
+    reflexManager: asFunction(({ eventBus, perceptionPipeline, logger }) =>
+      new ReflexManager({
+        eventBus,
+        perception: perceptionPipeline.getPerceptionAPI(),
+        logger,
+      }),
     ).singleton(),
   })
 

services/minecraft/src/cognitive/perception/perception-api.ts

@@ -0,0 +1,148 @@
+import type { Logg } from '@guiiai/logg'
+
+import type { Belief, EntityState, EntityView } from '../world/types'
+
+import { BeliefEngine } from '../semantics/belief-engine'
+import { teabagPattern } from '../semantics/patterns/teabag'
+import { EntityStore } from '../world/entity-store'
+import { TemporalBuffer } from '../world/temporal-buffer'
+
+/**
+ * Unified perception API for upper layers
+ * Provides entity queries and belief computations
+ */
+export class PerceptionAPI {
+  private store: EntityStore
+  private buffer: TemporalBuffer
+  private engine: BeliefEngine
+
+  constructor(
+    private readonly deps: {
+      logger: Logg
+    },
+  ) {
+    this.deps.logger.log('PerceptionAPI: initialized')
+    this.store = new EntityStore()
+    this.buffer = new TemporalBuffer(5000) // 5s history
+    this.engine = new BeliefEngine()
+
+    // Register default patterns
+    this.engine.register(teabagPattern)
+  }
+
+  // ============ Entity Updates (from Mineflayer) ============
+
+  /**
+   * Update an entity's state (called by perception collector)
+   */
+  updateEntity(id: string, partial: Partial<EntityState>): void {
+    const changes = this.store.update(id, partial)
+    this.buffer.recordAll(changes)
+  }
+
+  /**
+   * Remove an entity
+   */
+  removeEntity(id: string): void {
+    this.store.remove(id)
+    this.buffer.clearEntity(id)
+  }
+
+  /**
+   * Update self position (for distance calculations)
+   */
+  updateSelfPosition(x: number, y: number, z: number): void {
+    this.store.updateSelfPosition({ x, y, z } as any)
+  }
+
+  // ============ Entity Queries ============
+
+  /**
+   * Get all player entities with computed beliefs
+   */
+  getPlayers(): EntityView[] {
+    return this.store.getPlayers().map(e => this.buildEntityView(e))
+  }
+
+  /**
+   * Get a specific entity by ID
+   */
+  getEntity(id: string): EntityView | null {
+    const state = this.store.get(id)
+    if (!state)
+      return null
+    return this.buildEntityView(state)
+  }
+
+  // ============ Belief Queries ============
+
+  /**
+   * Find entities with high confidence of a pattern
+   */
+  entitiesWithBelief(pattern: string, minConfidence: number = 0.5): EntityView[] {
+    return this.getPlayers().filter(e => (e.beliefs[pattern]?.confidence ?? 0) >= minConfidence)
+  }
+
+  /**
+   * Get the top belief for an entity
+   */
+  getTopBelief(entityId: string): { pattern: string, belief: Belief } | null {
+    const entity = this.getEntity(entityId)
+    if (!entity)
+      return null
+
+    let top: { pattern: string, belief: Belief } | null = null
+    for (const [pattern, belief] of Object.entries(entity.beliefs)) {
+      if (!top || belief.confidence > top.belief.confidence) {
+        top = { pattern, belief }
+      }
+    }
+    return top
+  }
+
+  // ============ Pattern Management ============
+
+  /**
+   * Register a custom pattern
+   */
+  registerPattern(pattern: Parameters<BeliefEngine['register']>[0]): void {
+    this.engine.register(pattern)
+  }
+
+  // ============ Maintenance ============
+
+  /**
+   * Prune old history entries
+   */
+  prune(): void {
+    this.buffer.prune()
+  }
+
+  /**
+   * Clear all state
+   */
+  clear(): void {
+    this.store.clear()
+    this.buffer.clear()
+  }
+
+  // ============ Internal ============
+
+  private buildEntityView(state: EntityState): EntityView {
+    const beliefs = this.engine.computeBeliefs(
+      state.id,
+      id => this.store.get(id),
+      (id, since) => this.buffer.query(id, since),
+      this.store.getSelfPosition(),
+    )
+
+    return {
+      id: state.id,
+      name: state.name ?? state.id,
+      type: state.type,
+      state,
+      beliefs,
+      distanceToSelf: this.store.distanceToSelf(state.id) ?? Infinity,
+    }
+  }
+}

services/minecraft/src/cognitive/perception/pipeline.ts

@@ -11,10 +11,12 @@ import type { PerceptionStage } from './types/stage'
 import { DebugService } from '../../debug'
 import { createPerceptionFrameFromRawEvent } from './frame'
 import { MineflayerPerceptionCollector } from './mineflayer-perception-collector'
+import { PerceptionAPI } from './perception-api'
 import { SaliencyDetector } from './saliency-detector'
 
 export class PerceptionPipeline {
   private readonly detector: SaliencyDetector
+  private readonly perception: PerceptionAPI
   private collector: MineflayerPerceptionCollector | null = null
   private initialized = false
 
@@ -31,6 +33,8 @@ export class PerceptionPipeline {
       logger: Logg
     },
   ) {
+    this.perception = new PerceptionAPI({ logger: this.deps.logger })
+
     this.detector = new SaliencyDetector({
       logger: this.deps.logger,
       onAttention: (signal) => {
@@ -44,6 +48,31 @@ export class PerceptionPipeline {
     })
 
     this.stages = [
+      {
+        name: 'entity_update',
+        handle: (frame) => {
+          if (frame.kind !== 'world_raw')
+            return frame
+
+          const raw = frame.raw as RawPerceptionEvent
+
+          // Feed entity updates to PerceptionAPI
+          if ('entityId' in raw && 'entityType' in raw) {
+            const entityRaw = raw as RawPerceptionEvent & { entityId: string, entityType: string, displayName?: string, pos?: { x: number, y: number, z: number } }
+            if (entityRaw.entityType === 'player') {
+              this.perception.updateEntity(entityRaw.entityId, {
+                id: entityRaw.entityId,
+                type: 'player',
+                name: entityRaw.displayName,
+                position: entityRaw.pos as any,
+                isSneaking: 'sneaking' in entityRaw ? (entityRaw as any).sneaking : undefined,
+              })
+            }
+          }
+
+          return frame
+        },
+      },
       {
         name: 'attention',
         handle: (frame) => {
@@ -152,6 +181,13 @@ export class PerceptionPipeline {
     this.initialized = false
   }
 
+  /**
+   * Get the PerceptionAPI for querying entity beliefs
+   */
+  public getPerceptionAPI(): PerceptionAPI {
+    return this.perception
+  }
+
   public ingest(frame: PerceptionFrame): void {
     if (!this.initialized)
       return

services/minecraft/src/cognitive/reflex/behaviors/teabag.ts

@@ -5,22 +5,24 @@ export const teabagBehavior: ReflexBehavior = {
   modes: ['social', 'idle'],
   cooldownMs: 5000,
 
-  when: (ctx) => {
-    // Check if we recently received a teabag signal
-    // Check if we recently received a teabag signal
-    if (ctx.social.lastGesture === 'teabag') {
-      const now = Date.now()
-      const signalAge = now - (ctx.social.lastGestureAt || 0)
-
-      // Only respond if signal is fresh (< 2s)
-      return signalAge < 2000
-    }
-    return false
+  when: (_ctx, api) => {
+    // Check if any player is teabagging with high confidence
+    if (!api?.perception)
+      return false
+    const teabaggers = api.perception.entitiesWithBelief('teabag', 0.6)
+    return teabaggers.length > 0
   },
 
-  score: () => {
-    // Higher priority than LookAt (50)
-    return 60
+  score: (_ctx, api) => {
+    // Higher priority than LookAt (50), scaled by confidence
+    if (!api?.perception)
+      return 0
+    const teabaggers = api.perception.entitiesWithBelief('teabag', 0.6)
+    if (teabaggers.length === 0)
+      return 0
+    // Use highest confidence as score boost
+    const maxConfidence = Math.max(...teabaggers.map(e => e.beliefs.teabag?.confidence ?? 0))
+    return 60 + (maxConfidence * 20)
   },
 
   run: async ({ bot }) => {

services/minecraft/src/cognitive/reflex/reflex-manager.test.ts

@@ -43,7 +43,14 @@ describe('reflexManager', () => {
     } as any
 
     const logger = makeLogger()
-    const reflex = new ReflexManager({ eventBus, logger }) // Now accepts eventBus
+    const perception = {
+      getPlayers: vi.fn(() => []),
+      getEntity: vi.fn(() => null),
+      entitiesWithBelief: vi.fn(() => []),
+      updateEntity: vi.fn(),
+      updateSelfPosition: vi.fn(),
+    } as any
+    const reflex = new ReflexManager({ eventBus, perception, logger })
 
     const bot = makeBot()
     reflex.init(bot)

services/minecraft/src/cognitive/reflex/reflex-manager.ts

@@ -1,6 +1,7 @@
 import type { Logg } from '@guiiai/logg'
 
 import type { EventBus, TracedEvent } from '../os'
+import type { PerceptionAPI } from '../perception/perception-api'
 import type { PerceptionSignal } from '../perception/types/signals'
 import type { MineflayerWithAgents } from '../types'
 import type { ReflexContextState } from './context'
@@ -19,6 +20,7 @@ export class ReflexManager {
   constructor(
     private readonly deps: {
       eventBus: EventBus
+      perception: PerceptionAPI
       logger: Logg
     },
   ) {
@@ -86,7 +88,7 @@ export class ReflexManager {
     // For greeting behavior compatibility, we might need to map specific signals to social state.
 
     // Trigger behavior selection
-    this.runtime.tick(bot, 0)
+    this.runtime.tick(bot, 0, this.deps.perception)
 
     // Emit reflex state for observability
     DebugService.getInstance().emitReflexState({

services/minecraft/src/cognitive/reflex/runtime.ts

@@ -1,5 +1,6 @@
 import type { Logg } from '@guiiai/logg'
 
+import type { PerceptionAPI } from '../perception/perception-api'
 import type { MineflayerWithAgents } from '../types'
 import type { ReflexModeId } from './modes'
 import type { ReflexBehavior } from './types/behavior'
@@ -38,7 +39,7 @@ export class ReflexRuntime {
     this.behaviors.push(behavior)
   }
 
-  public tick(bot: MineflayerWithAgents, deltaMs: number): string | null {
+  public tick(bot: MineflayerWithAgents, deltaMs: number, perception: PerceptionAPI): string | null {
     const now = Date.now()
 
     this.context.updateNow(now)
@@ -73,16 +74,17 @@ export class ReflexRuntime {
     this.activeBehaviorUntil = null
 
     const ctx = this.context.getSnapshot()
+    const api = { bot, context: this.context, perception }
 
     let best: { behavior: ReflexBehavior, score: number } | null = null
     for (const behavior of this.behaviors) {
       if (!behavior.modes.includes(this.mode))
         continue
 
-      if (!behavior.when(ctx))
+      if (!behavior.when(ctx, api))
         continue
 
-      const score = behavior.score(ctx)
+      const score = behavior.score(ctx, api)
       if (score <= 0)
         continue
 
@@ -102,7 +104,7 @@ export class ReflexRuntime {
     this.runHistory.set(best.behavior.id, { lastRunAt: now })
 
     try {
-      const maybePromise = best.behavior.run({ bot, context: this.context })
+      const maybePromise = best.behavior.run(api)
       if (maybePromise && typeof (maybePromise as any).then === 'function') {
         this.activeBehaviorUntil = now + Math.max(deltaMs, 50)
         void (maybePromise as Promise<void>).finally(() => {