Merge pull request #3 from Tesla2000/feature/evaluation

Feature/evaluation

Author: Tesla2000

.gitignore

@@ -1,3 +1,5 @@
 /sandbox.py
 /data/
 /models/
+/evaluation_data/
+/training_data/

Config.py

@@ -7,33 +7,42 @@
 
 class _ConfigPaths:
     root = Path(__file__).parent
-    data_path = root / 'data'
-    data_path.mkdir(exist_ok=True)
-    model_path = root / 'models'
+    training_data_path = root / "training_data"
+    training_data_path.mkdir(exist_ok=True)
+    evaluation_data_path = root / "evaluation_data"
+    evaluation_data_path.mkdir(exist_ok=True)
+    model_path = root / "models"
     model_path.mkdir(exist_ok=True)
 
 
 class _ConfigAgent:
-    # hidden_sizes = (256, 128, 64, 32)
-    hidden_sizes = (256,)
-    # hidden_sizes = tuple()
-    c = .1
-    learning_rate = 1e-3
+    # hidden_sizes = (
+    #     256,
+    #     128,
+    #     64,
+    #     32,
+    # )
+    # hidden_sizes = (256,)
+    hidden_sizes = tuple()
+    c = 0.2
+    learning_rate = 1e-5
     debug = False
     pretrain = True
 
 
 class Config(_ConfigPaths, _ConfigAgent):
+    train = False
     max_results_held = 100
     minimal_relative_agent_improvement = 1.1
-    min_games_to_replace_agents = 20
-    train_batch_size = 64
-    training_buffer_len = 1000
+    min_games_to_replace_agents = 40
+    train_batch_size = 128
+    training_buffer_len = 100_000
     min_n_points_to_finish = 15
     n_simulations = 100
     n_games = None
     n_players = 2
-    n_actions = 46
+    n_actions = 45
+    eval_rate = 0.2
 
 
 if Config.debug:

agent/Agent.py

@@ -15,7 +15,7 @@ def __init__(
         self,
         n_players: int,
         hidden_sizes: tuple = Config.hidden_sizes,
-        n_moves: int = 46,
+        n_moves: int = Config.n_actions,
     ):
         super().__init__()
         self.tanh = nn.Tanh()

agent/RLDataset.py

@@ -1,16 +1,19 @@
-from collections import deque
+from typing import Sequence
 
 import numpy as np
 from torch.utils.data import Dataset
 
 
 class RLDataset(Dataset):
-    def __init__(self, examples: deque[tuple[tuple, np.array, int]]):
+    def __init__(self, examples: Sequence[tuple[tuple, np.array, int]]):
         self.examples = examples
 
     def __len__(self):
         return len(self.examples)
 
     def __getitem__(self, index) -> tuple[np.array, ...]:
-        return np.array(self.examples[index][0]), np.array(self.examples[index][1]), np.array(
-            [self.examples[index][2] * 2 - 1])
+        return (
+            np.array(self.examples[index][0]),
+            np.array(self.examples[index][1]),
+            np.array([self.examples[index][2] * 2 - 1]),
+        )

agent/pretrain.py

@@ -0,0 +1,40 @@
+import operator
+from collections import deque
+from functools import reduce
+from itertools import islice
+
+import torch
+
+from Config import Config
+from agent.Agent import Agent
+from agent.train_agent import train_agent
+
+
+def pretrain(agents: deque[Agent]):
+    for agent, checkpoint_index in zip(
+        islice(reversed(agents), 1, None),
+        sorted(
+            (int(path.name.split(".")[0]) for path in Config.model_path.iterdir()),
+            reverse=True,
+        ),
+    ):
+        agent.load_state_dict(
+            torch.load(Config.model_path.joinpath(f"{checkpoint_index}.pth"))
+        )
+    newest = Config.model_path.joinpath(
+        f"{max((*tuple(int(path.name.split('.')[0]) for path in Config.model_path.iterdir()), 0))}.pth"
+    )
+    if newest.exists():
+        agents[-1].load_state_dict(torch.load(newest))
+    training_buffer = reduce(
+        operator.add,
+        (
+            deque(eval(path.read_text()))
+            for path in sorted(
+                Config.training_data_path.iterdir(), key=lambda path: int(path.name)
+            )
+        ),
+        deque(maxlen=Config.training_buffer_len),
+    )
+    train_agent(agents[-1], training_buffer)
+    return training_buffer

agent/save.py

@@ -0,0 +1,9 @@
+from Config import Config
+
+
+def save_temp_buffer(buffer, train: bool):
+    path = Config.training_data_path if train else Config.evaluation_data_path
+    index = max((*tuple(int(path.name) for path in path.iterdir()), -1)) + 1
+    path.joinpath(str(index)).write_text(
+        str(list((list(sample[0]), list(sample[1]), sample[2]) for sample in buffer))
+    )

agent/search.py

@@ -40,4 +40,4 @@ def search(
         N[state][action] + 1
     )
     N[state][action] += 1
-    return -v
+    return -v

agent/self_play.py

@@ -1,30 +1,58 @@
+import random
 from collections import deque
 from itertools import count
 
 import numpy as np
 from tqdm import tqdm
 
 from Config import Config
+from src.Game import Game
 from .Agent import Agent
 from .policy import policy
-from src.Game import Game
 
 
-def self_play(agents: deque[Agent]) -> tuple[list[tuple[np.array, np.array, int]], Agent]:
+def self_play(
+    agents: deque[Agent],
+) -> tuple[list[tuple[np.array, np.array, int]], list[Agent]]:
     states = []
+    winners = []
     game = Game(n_players=Config.n_players)
-    id_to_agent = dict((player.id, agent) for agent, player in zip(agents, game.players))
     for agent in agents:
         agent.eval()
-    for _ in tqdm(count()):
+    id_to_agent = dict(
+        (player.id, agent)
+        for agent, player in zip(random.sample(agents, Config.n_players), game.players)
+    )
+    results, winner = _perform_game(game, [], id_to_agent)
+    states += results
+    winners.append(winner)
+    return states, winners
+
+
+def _perform_game(
+    game: Game, states: list, id_to_agent: dict[int, Agent]
+) -> tuple[list[tuple[np.array, np.array, int]], Agent]:
+    for turn in tqdm(count()):
         agent = id_to_agent[game.current_player.id]
         pi, action = policy(game, agent, Config.c, Config.n_simulations)
         action_index = game.all_moves.index(action)
         onehot_encoded_action = np.zeros(Config.n_actions)
         onehot_encoded_action[action_index] = 1
-        states.append((game, onehot_encoded_action, 0))
+        states.append((game, action, 0))
         game = game.perform(action)
         if game.is_terminal():
             result = game.get_results()
-            return (list((state[0].get_state(), state[1], int(result[state[0].current_player.id] == 1)) for state in states),
-                    id_to_agent[next(player.id for player in game.players if result[player.id])])
+            return (
+                list(
+                    (
+                        state[0].get_state(),
+                        np.eye(Config.n_actions)[game.all_moves.index(state[1])],
+                        int(result[state[0].current_player.id] == 1),
+                    )
+                    for state in states
+                    if state[1] != game.null_move
+                ),
+                id_to_agent[
+                    next(player.id for player in game.players if result[player.id])
+                ],
+            )

agent/train_agent.py

@@ -1,6 +1,7 @@
-from collections import deque
+from typing import Sequence
 
 import numpy as np
+from sklearn.metrics import accuracy_score
 from torch import nn, optim
 from torch.utils.data import DataLoader
 
@@ -9,20 +10,49 @@
 from .RLDataset import RLDataset
 
 
-def train_agent(agent: Agent, train_data: deque[tuple[tuple, np.array, int]]):
+def train_agent(agent: Agent, train_data: Sequence[tuple[tuple, np.array, int]]):
     agent.train()
+    optimizer = optim.Adam(agent.parameters(), lr=Config.learning_rate)
+    _loop(agent, train_data, optimizer)
+
+
+def eval_agent(agent: Agent, eval_set: Sequence[tuple[tuple, np.array, int]]):
+    agent.eval()
+    return _loop(agent, eval_set, batch_size=len(eval_set))
+
+
+def _loop(
+    agent: Agent,
+    dataset: Sequence[tuple[tuple, np.array, int]],
+    optimizer: optim.Optimizer = None,
+    batch_size=Config.train_batch_size,
+):
+    is_optimizer = optimizer is not None
     categorical_cross_entropy = nn.CrossEntropyLoss()
     mse = nn.MSELoss()
-    optimizer = optim.Adam(agent.parameters(), lr=Config.learning_rate)
-    dataset = RLDataset(train_data)
-    loader = DataLoader(dataset, batch_size=Config.train_batch_size)
-    for batch in loader:
-        state, policy, win_probability = batch
-        state, policy, win_probability = state.float(), policy.float(), win_probability.float()
-        optimizer.zero_grad()
+    dataset = RLDataset(dataset)
+    loader = DataLoader(dataset, batch_size=batch_size)
+    for index, (state, policy, win_probability) in enumerate(loader):
+        state, policy, win_probability = (
+            state.float(),
+            policy.float(),
+            win_probability.float(),
+        )
+        if is_optimizer:
+            optimizer.zero_grad()
         output_policy, output_v = agent(state)
         bce = mse(output_v, win_probability)
         cce = categorical_cross_entropy(output_policy, policy)
-        bce.backward(retain_graph=True)
-        cce.backward()
-        optimizer.step()
+        if is_optimizer:
+            bce.backward(retain_graph=True)
+            cce.backward()
+            optimizer.step()
+        else:
+            print(
+                accuracy_score(win_probability, np.sign(output_v.detach().numpy())),
+                accuracy_score(
+                    np.argmax(policy.detach().numpy(), axis=1),
+                    np.argmax(output_policy.detach().numpy(), axis=1),
+                ),
+            )
+            return bce.item(), cce.item()