181 lines
6.4 KiB
Python
181 lines
6.4 KiB
Python
from collections import deque
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
import GEO_parameters
|
|
from Replay_memory import Memory, Transaction
|
|
from model import Net
|
|
import torch.nn.functional as f
|
|
import matplotlib.pyplot as plt
|
|
from GEO_parameters import GEO_Parameters
|
|
|
|
|
|
def dev0(a, b):
|
|
a = a.detach().numpy()
|
|
b = b.detach().numpy()
|
|
with np.errstate(divide='ignore', invalid='ignore'):
|
|
c = np.true_divide(a, b)
|
|
c[~ np.isfinite(c)] = 0
|
|
c = torch.from_numpy(c)
|
|
return c
|
|
|
|
|
|
def soft_update(target, source, tau):
|
|
for target_param, param in zip(target.parameters(), source.parameters()):
|
|
target_param.data.copy_(target_param.data * (1.0 - tau) + param.data * tau)
|
|
|
|
|
|
def hard_update(target, source):
|
|
for target_param, param in zip(target.parameters(), source.parameters()):
|
|
target_param.data.copy_(param.data)
|
|
|
|
|
|
class DeepQNetwork:
|
|
def __init__(self, cap, n_actions, n_states, learning_rate, reward_decay, e_greedy, replace_target_iter, batch_size,e_greedy_increment=None):
|
|
|
|
self.memory = Memory(cap)
|
|
self.batch = None
|
|
self.state_batch = batch_size
|
|
self.action_batch = None
|
|
self.reward_batch = None
|
|
self.non_final_next_states = None
|
|
self.expected_state_action_values = None
|
|
self.state_action_values = None
|
|
|
|
self.n_states = n_states
|
|
self.n_actions = n_actions
|
|
self.lr = learning_rate
|
|
self.gamma = reward_decay
|
|
self.epsilon_max = e_greedy
|
|
self.replace_target_iter = replace_target_iter
|
|
self.batch_size = batch_size
|
|
self.epsilon_decay = e_greedy_increment
|
|
self.epsilon = 0 if e_greedy_increment is None else self.epsilon_max
|
|
|
|
|
|
self.learn_step_counter = 1
|
|
|
|
self.counter = GEO_parameters.state_dim
|
|
self.main_q_network = Net(n_states, n_actions) # 输入的nstates 应该是M_c+1
|
|
self.target_q_network = Net(n_states, n_actions)
|
|
|
|
self.optimizer = torch.optim.Adam(self.main_q_network.parameters(), self.lr)
|
|
|
|
self.cost_his = []
|
|
|
|
self.weight_dir = 'weight_DQN'
|
|
|
|
def replay(self):
|
|
if self.learn_step_counter % self.replace_target_iter == 0:
|
|
self.update_target_q_network()
|
|
|
|
if len(self.memory) <self.batch_size:
|
|
return
|
|
|
|
self.batch, self.state_batch, self.action_batch, self.reward_batch, self.non_final_next_states = self.make_mini_batch()
|
|
self.expected_state_action_values = self.get_expected_state_action_values()
|
|
self.update_main_q_network()
|
|
self.epsilon *= 0.9998
|
|
self.learn_step_counter += 1
|
|
|
|
def make_mini_batch(self):
|
|
transitions = self.memory.sample(self.batch_size)
|
|
|
|
batch = Transaction(*zip(*transitions))
|
|
state_batch = torch.cat(batch.state)
|
|
action_batch = torch.cat(batch.action)
|
|
reward_batch = torch.cat(batch.reward)
|
|
non_final_next_states = torch.cat([s for s in batch.next_state if s is not None])
|
|
return batch, state_batch, action_batch, reward_batch, non_final_next_states
|
|
|
|
def get_expected_state_action_values(self):
|
|
self.main_q_network.eval()
|
|
self.target_q_network.eval()
|
|
|
|
self.state_action_values = self.main_q_network
|
|
|
|
non_final_mask = torch.ByteTensor(tuple(map(lambda x : x is not None, self.batch.next_state))).bool()
|
|
|
|
next_state_values = torch.zeros(self.batch_size)
|
|
|
|
a_m = torch.zeros(self.batch_size).type(torch.LongTensor)
|
|
|
|
a_m[non_final_mask] = self.main_q_network(self.non_final_next_states).detach().max(1)[1]
|
|
|
|
a_m_non_final_next_states = a_m[non_final_mask].view(-1, 1)
|
|
|
|
next_state_values[non_final_mask] = self.target_q_network(
|
|
self.non_final_next_states
|
|
).gather(1, a_m_non_final_next_states).detach().squeeze()
|
|
|
|
expected_state_action_values = self.reward_batch + self.gamma * next_state_values.view(-1, 1)
|
|
return expected_state_action_values
|
|
|
|
def decide_action(self, state):
|
|
# self.epsilon = 0.5 * (1 / (self.epsilon + 1))
|
|
if self.epsilon < np.random.uniform():
|
|
self.main_q_network.eval()
|
|
with torch.no_grad():
|
|
action = self.main_q_network(state).detach().max(1)[1]
|
|
else:
|
|
action = self.LongTenser([np.random.randint(0, self.n_actions)])
|
|
return action.view(1, 1)
|
|
|
|
def update_main_q_network(self):
|
|
self.main_q_network.train()
|
|
|
|
loss = f.mse_loss(self.state_action_values, self.expected_state_action_values, reduction = 'mean')
|
|
self.cost_his.append(loss.item())
|
|
self.optimizer.zero_grad()
|
|
loss.backward()
|
|
self.optimizer.step()
|
|
|
|
def update_target_q_network(self):
|
|
self.target_q_network.load_state_dict(self.main_q_network.state_dict())
|
|
|
|
def plot_cost(self):
|
|
plt.plot(np.arange(len(self.cost_his)), self.cost_his)
|
|
plt.xlabel('training step')
|
|
plt.ylabel('cost')
|
|
plt.show()
|
|
|
|
def binary(self, q):
|
|
self.counter = GEO_Parameters.state_dim - 1
|
|
t = torch.unsqueeze(q.max(1)[1], 1)
|
|
temp = torch.zeros_like(t)
|
|
temp_extend = temp.repeat(1, GEO_Parameters.state_dim)
|
|
n = temp
|
|
while torch.logical_not(torch.eq(n, t)):
|
|
t1 = t % 2
|
|
t2 = t >> 1
|
|
numColumns = temp_extend.shape[1]
|
|
new_tensor_left = temp_extend[:, :self.counter]
|
|
new_tensor_right = temp_extend[:, self.counter+1:numColumns]
|
|
temp_extend = torch.cat([new_tensor_left, new_tensor_right], 1)
|
|
self.counter -= 1
|
|
|
|
a1 = temp_extend
|
|
a1 = a1.float()
|
|
return a1
|
|
|
|
def load_weight_from_pkl(self, file_name):
|
|
checkpoint = torch.load(file_name)
|
|
self.main_q_network.load_state_dict(checkpoint['main_q_network_state_dict'])
|
|
self.target_q_network.load_state_dict(checkpoint['target_q_network_state_dict'])
|
|
self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
|
|
self.cost_his = checkpoint['loss']
|
|
|
|
def save_weight_to_pkl(self, file_name):
|
|
torch.save({
|
|
# 'epoch': epochs,
|
|
'main_q_network_state_dict': self.main_q_network.state_dict(),
|
|
'target_q_network_state_dict': self.target_q_network.state_dict(),
|
|
'optimizer_state_dict': self.optimizer.state_dict(),
|
|
'loss': self.cost_his,
|
|
}, file_name)
|
|
|
|
def update_replay_buffer(self, state, action, reward, next_state):
|
|
self.memory.push(state, action, reward, next_state)
|
|
|