76 lines
1.9 KiB
Python
76 lines
1.9 KiB
Python
import numpy as np
|
|
|
|
from Environment import Env
|
|
|
|
np.random.seed(1)
|
|
|
|
|
|
class DynaQ:
|
|
def __init__(self, env: Env, episodes: int, epsilon: float, alpha: float, gamma: float, n_steps: int):
|
|
# Initialize parameter
|
|
self.env = env
|
|
self.alpha = alpha
|
|
self.gamma = gamma
|
|
self.epsilon = epsilon
|
|
self.episodes = episodes
|
|
self.n_steps = n_steps
|
|
|
|
self.time_step = 0
|
|
self.state = self.env.start
|
|
self.steps_per_episode = []
|
|
self.state_actions = []
|
|
self.step_in_episode = 0
|
|
|
|
# Initialize Q-matrix and model
|
|
self.Q = {}
|
|
self.model = {}
|
|
|
|
for state in list(self.env.G):
|
|
self.Q[state] = {}
|
|
self.model[state] = {}
|
|
for action in list(self.env.G.neighbors(state)) + [state]:
|
|
self.Q[state][action] = 0
|
|
self.model[state][action] = (-1, action, 0)
|
|
|
|
'''
|
|
Resets the model
|
|
'''
|
|
|
|
def reset(self) -> None:
|
|
self.state = self.env.start
|
|
self.state_actions = []
|
|
self.step_in_episode = 0
|
|
self.env.reset()
|
|
|
|
'''
|
|
Learning method for agent
|
|
Basically DynaQ algorithm adapted for graphs
|
|
'''
|
|
|
|
def learn(self, epsilon_decay: float, epsilon_min: float, run: int) -> None:
|
|
# todo: implement learning
|
|
pass
|
|
|
|
'''
|
|
Returns epsilon-greedy action
|
|
'''
|
|
|
|
def get_action(self, eps: float) -> int:
|
|
# todo: implement eval
|
|
pass
|
|
|
|
'''
|
|
Add Reward, next state and current time step to state-action pair in model
|
|
'''
|
|
|
|
def update_model(self, state: int, action: int, reward: float, next_state) -> None:
|
|
self.model[state][action] = (reward, next_state, self.time_step)
|
|
|
|
'''
|
|
Planning phase, basically Bellmann equation with already taken state-action pairs
|
|
'''
|
|
|
|
def planning(self) -> None:
|
|
# todo: implement planning
|
|
pass
|