modifying options to find a mistake

Author: Tesla2000

Config.py

@@ -30,7 +30,7 @@ class Config(_ConfigPaths, _ConfigAgent):
     train_batch_size = 64
     training_buffer_len = 1000
     min_n_points_to_finish = 15
-    n_simulations = 100
+    n_simulations = 250
     n_games = None
     n_players = 2
     n_actions = 46

agent/RLDataset.py

@@ -12,4 +12,5 @@ def __len__(self):
         return len(self.examples)
 
     def __getitem__(self, index) -> tuple[np.array, ...]:
-        return np.array(self.examples[index][0]), np.array(self.examples[index][1]), np.array([self.examples[index][2]])
+        return np.array(self.examples[index][0]), np.array(self.examples[index][1]), np.array(
+            [self.examples[index][2] * 2 - 1])

agent/policy.py

@@ -9,7 +9,7 @@
 
 def policy(
     game: Game,
-    agents: dict[int, nn.Module],
+    agent: nn.Module,
     c: float,
     n_simulations: int,
 ):
@@ -20,6 +20,6 @@ def policy(
     initial_state = game.get_state()
     all_moves = game.get_possible_actions()
     for _ in range(n_simulations):
-        search(game.copy(), agents, c, N, visited, P, Q)
+        search(game.copy(), agent, c, N, visited, P, Q)
     pi = [N[initial_state][a] for a in all_moves]
     return pi, all_moves[np.argmax(pi)]

agent/search.py

@@ -8,17 +8,16 @@
 
 def search(
     game: Game,
-    agents: dict[int, nn.Module],
+    agent: nn.Module,
     c: float,
     N: defaultdict,
     visited: set,
     P: defaultdict,
     Q: defaultdict,
 ):
     if game.is_terminal():
-        return game.get_results()[game.current_player.id]
+        return -game.get_results()[game.current_player.id]
     state = game.get_state()
-    agent = agents[game.current_player.id]
     if state not in visited:
         visited.add(state)
         move_scores, v = agent(Tensor([state]))
@@ -35,7 +34,7 @@ def search(
     )
 
     next_game_state = game.perform(action)
-    v = search(next_game_state, agents, c, N, visited, P, Q)
+    v = search(next_game_state, agent, c, N, visited, P, Q)
 
     Q[state][action] = (N[state][action] * Q[state].get(action, 1) + v) / (
         N[state][action] + 1

agent/self_play.py

@@ -17,7 +17,8 @@ def self_play(agents: deque[Agent]) -> tuple[list[tuple[np.array, np.array, int]
     for agent in agents:
         agent.eval()
     for _ in tqdm(count()):
-        pi, action = policy(game, id_to_agent, Config.c, Config.n_simulations)
+        agent = id_to_agent[game.current_player.id]
+        pi, action = policy(game, agent, Config.c, Config.n_simulations)
         action_index = game.all_moves.index(action)
         onehot_encoded_action = np.zeros(Config.n_actions)
         onehot_encoded_action[action_index] = 1

agent/train_agent.py

@@ -12,7 +12,7 @@
 def train_agent(agent: Agent, train_data: deque[tuple[tuple, np.array, int]]):
     agent.train()
     categorical_cross_entropy = nn.CrossEntropyLoss()
-    binary_cross_entropy = nn.BCELoss()
+    mse = nn.MSELoss()
     optimizer = optim.Adam(agent.parameters(), lr=Config.learning_rate)
     dataset = RLDataset(train_data)
     loader = DataLoader(dataset, batch_size=Config.train_batch_size)
@@ -21,7 +21,7 @@ def train_agent(agent: Agent, train_data: deque[tuple[tuple, np.array, int]]):
         state, policy, win_probability = state.float(), policy.float(), win_probability.float()
         optimizer.zero_grad()
         output_policy, output_v = agent(state)
-        bce = binary_cross_entropy((output_v + 1) / 2, win_probability)
+        bce = mse(output_v, win_probability)
         cce = categorical_cross_entropy(output_policy, policy)
         bce.backward(retain_graph=True)
         cce.backward()