Added missing state values

Author: Tesla2000

Config.py

@@ -27,7 +27,8 @@ class _ConfigAgent:
     c = 0.2
     learning_rate = 1e-5
     debug = False
-    pretrain = True
+    # pretrain = True
+    pretrain = False
 
 
 class Config(_ConfigPaths, _ConfigAgent):

agent/Agent.py

@@ -8,7 +8,7 @@
 
 class Agent(nn.Module):
     _input_size_dictionary = {
-        2: 205,
+        2: 211,
     }
 
     def __init__(

agent/policy.py

@@ -13,10 +13,10 @@ def policy(
     c: float,
     n_simulations: int,
 ):
-    N = defaultdict(lambda: defaultdict(int))
+    N = defaultdict(list)
     visited = set()
-    P = defaultdict(dict)
-    Q = defaultdict(dict)
+    P = defaultdict(list)
+    Q = defaultdict(list)
     initial_state = game.get_state()
     all_moves = game.get_possible_actions()
     for _ in range(n_simulations):

agent/search.py

@@ -10,10 +10,10 @@ def search(
     game: Game,
     agent: nn.Module,
     c: float,
-    N: defaultdict,
+    N: defaultdict[list[int]],
     visited: set,
-    P: defaultdict,
-    Q: defaultdict,
+    P: defaultdict[list],
+    Q: defaultdict[list],
 ):
     if game.is_terminal():
         return -game.get_results()[game.current_player.id]
@@ -26,13 +26,26 @@ def search(
             for index, move in enumerate(game.all_moves)
         )
         return -v
-
-    action = max(
-        game.get_possible_actions(),
-        key=lambda action: Q[state].get(action, 1)
-        + c * P[state][action] * sqrt(sum(N[state].values())) / (1 + N[state][action]),
-    )
-
+    q_state = Q[state]
+    p_state = P[state]
+    n_state = N[state]
+    sqrt_value = sqrt(sum(n_state.values()))
+    def _get_action(game: Game):
+        return max(
+            game.get_possible_actions(),
+            key=lambda action: q_state.get(action, 1) + c * p_state[action] * sqrt_value / (1 + n_state[action]),
+        )
+    # def _get_action(game: Game):
+    #     actions = sorted(
+    #         game.all_moves,
+    #         key=lambda action: q_state.get(action, 1)
+    #         + c * p_state[action] * sqrt_value / (1 + n_state[action]),
+    #         reverse=True,
+    #     )
+    #     for action in actions:
+    #         if action.is_valid(game):
+    #             return action
+    action = _get_action(game)
     next_game_state = game.perform(action)
     v = search(next_game_state, agent, c, N, visited, P, Q)
 

src/Game.py

@@ -155,6 +155,11 @@ def get_possible_actions(self) -> tuple[Move, ...]:
             (self.null_move,) if self.null_move.is_valid(self) else tuple()
         )
 
+    def get_possible_action_indexes(self) -> tuple[int, ...]:
+        return tuple(index for index, move in enumerate(self.all_moves) if move.is_valid(self)) or (
+            (self.null_move,) if self.null_move.is_valid(self) else tuple()
+        )
+
     combos = combinations([{field.name: 1} for field in fields(BasicResources)], 3)
     all_moves = list(
         GrabThreeResource(BasicResources(**res_1, **res_2, **res_3))

src/StateExtractor.py

@@ -48,6 +48,7 @@ def get_state(cls, game: "Game") -> tuple:
                         tuple(iter(aristocrat.cost))
                         for aristocrat in game.board.aristocrats
                     ),
+                    iter(game.board.resources),
                     chain.from_iterable(
                         (
                             *tuple(iter(player.resources)),