It is running but is it slow...

Author: Tesla2000

Config.py

@@ -18,6 +18,11 @@ class _ConfigPaths:
 
 
 class Config(_ConfigPaths):
+    # hidden_sizes = (256, 128, 64, 32)
+    hidden_sizes = (256,)
+    # hidden_sizes = tuple()
+    learning_rate = 1e-3
+    max_results_held = 100
     minimal_relative_agent_improvement = 1.1
     min_games_to_replace_agents = 20
     train_batch_size = 64

agent/Agent.py

@@ -1,7 +1,10 @@
 from itertools import pairwise, starmap
 
+import numpy as np
 from torch import nn, Tensor
 
+from Config import Config
+
 
 class Agent(nn.Module):
     _input_size_dictionary = {
@@ -11,27 +14,29 @@ class Agent(nn.Module):
     def __init__(
         self,
         n_players: int,
-        hidden_sizes: tuple = (256, 128, 64, 32),
+        hidden_sizes: tuple = Config.hidden_sizes,
         n_moves: int = 46,
     ):
         super().__init__()
-        self.relu = nn.ReLU()
         self.tanh = nn.Tanh()
         self.softmax = nn.Softmax(dim=1)
         first_size = self._get_size(n_players)
         sizes = first_size, *hidden_sizes
-        self.layers = tuple(starmap(nn.Linear, pairwise(sizes)))
-        for index, layer in enumerate(self.layers):
-            setattr(self, f"layer_{index}", layer)
-        self.fc_v = nn.Linear(hidden_sizes[-1], 1)
-        self.fc_p = nn.Linear(hidden_sizes[-1], n_moves)
+        self.layers = nn.ModuleList(starmap(nn.Linear, pairwise(sizes)))
+        self.trained = False
+        self.fc_v = nn.Linear(sizes[-1], 1)
+        self.fc_p = nn.Linear(sizes[-1], n_moves)
         self._n_moves = n_moves
 
     def _get_size(self, n_players: int) -> int:
         return self._input_size_dictionary[n_players]
 
     def forward(self, state: Tensor):
+        if not self.training and not self.trained:
+            return self.softmax(Tensor(np.random.random((1, self._n_moves)))), Tensor(
+                np.random.uniform(-1, 1, (1, 1))
+            )
+        self.trained = True
         for layer in self.layers:
             state = layer(state)
-            state = self.relu(state)
         return self.softmax(self.fc_p(state)), self.tanh(self.fc_v(state))

agent/train_agent.py

@@ -13,7 +13,7 @@ def train_agent(agent: Agent, train_data: deque[tuple[tuple, np.array, int]]):
     agent.train()
     categorical_cross_entropy = nn.CrossEntropyLoss()
     binary_cross_entropy = nn.BCELoss()
-    optimizer = optim.Adam(agent.parameters())
+    optimizer = optim.Adam(agent.parameters(), lr=Config.learning_rate)
     dataset = RLDataset(train_data)
     loader = DataLoader(dataset, batch_size=Config.train_batch_size)
     for batch in loader:

main.py

@@ -14,15 +14,16 @@
 def main():
     training_buffer = deque(maxlen=Config.training_buffer_len)
     agents = deque((Agent(Config.n_players) for _ in range(Config.n_players)), maxlen=Config.n_players)
-    agent_scores = defaultdict(int, ((id(agent), 0) for agent in agents))
+    scores = deque(maxlen=Config.max_results_held)
     for _ in (count() if Config.n_games is None else range(Config.n_games)):
-        buffer, agent = self_play(agents)
-        agent_scores[id(agent)] += 1
-        if sum(agent_scores.values()) >= Config.min_games_to_replace_agents and agent_scores[id(agents[-1])] > Config.minimal_relative_agent_improvement * sum(agent_scores.values()) / len(agents):
-            torch.save(agent[-1].state_dict(), Config.model_path.joinpath(str(max(map(int, (*re.findall(r'\d+', ''.join(Config.model_path.iterdir())), -1))) + 1) + ".pth"))
-            agents.append(Agent(Config.n_players).load_state_dict(deepcopy(agent[-1].state_dict())))
-            agent_scores = defaultdict(int, ((id(agent), 0) for agent in agents))
-        print(agent_scores[id(agents[-1])], sum(agent_scores.values()), agent_scores[id(agents[-1])] / sum(agent_scores.values()))
+        buffer, winner = self_play(agents)
+        scores.append(agents[-1] is winner)
+        if len(scores) >= Config.min_games_to_replace_agents and sum(scores) > Config.minimal_relative_agent_improvement * len(scores) / len(agents):
+            torch.save(agents[-1].state_dict(), Config.model_path.joinpath(str(max(map(int, (*re.findall(r'\d+', ''.join(Config.model_path.iterdir())), -1))) + 1) + ".pth"))
+            agents.append(Agent(Config.n_players).load_state_dict(deepcopy(agents[-1].state_dict())))
+            agents[-1].training = True
+            scores = deque(maxlen=Config.max_results_held)
+        print(sum(scores) / len(scores), len(scores))
         training_buffer += buffer
         train_agent(agents[-1], training_buffer)
 

src/moves/GrabTwoResource.py

@@ -20,7 +20,7 @@ def perform(self, game: "Game") -> "Game":
         return game
 
     def is_valid(self, game: "Game") -> bool:
-        resource = next(compress(asdict(self.resources).keys(), iter(self.resources)))
+        resource = next(compress(('red', 'green', 'blue', 'black', 'white'), iter(self.resources)))
         if getattr(game.board.resources, resource) < 4:
             return False
         return super().is_valid(game)