From ea146f8302f9148763c7d5fcbb6bf134dfdb8a8b Mon Sep 17 00:00:00 2001
From: Junior <jun.gao.work@outlook.com>
Date: Wed, 6 Mar 2024 22:40:09 +0800
Subject: [PATCH] frameworks

---
 GEOEnv.py         |  94 ++++++++++++++++++++++++++++
 GEO_parameters.py |  13 ++++
 dqn.py            | 155 ++++++++++++++++++++++++++++++++++++++++++++++
 drl_framework.py  |  29 +++++++++
 model.py          |   5 ++
 5 files changed, 296 insertions(+)
 create mode 100644 GEOEnv.py
 create mode 100644 GEO_parameters.py
 create mode 100644 drl_framework.py

diff --git a/GEOEnv.py b/GEOEnv.py
new file mode 100644
index 0000000..2023236
--- /dev/null
+++ b/GEOEnv.py
@@ -0,0 +1,94 @@
+import numpy as np
+from GEO_parameters import GEO_Parameters
+import torch
+
+
+class GEOEnv:
+    def __init__(self, num_satellite = 1, num_carrier_wave = 100, num_beam = 1, num_user_dim = 32, delt_t =5, dataset = "month_2.txt", max_intervals = 8):
+        self.action_space = num_carrier_wave
+        self.user_space_dim = num_user_dim
+        self.user_space = num_user_dim * num_user_dim
+        self.delt_t =  delt_t
+        self.beams = num_beam
+        self.satellite = num_satellite
+        self.steps_before_termination = 0
+        self.datas = []
+        self.dataset = dataset
+        self.num_data = len(self.datas)
+        self.state = np.zeros((self.action_space+1, self.user_space_dim, self.user_space_dim), dtype=np.uint8)
+        self.max_intervals = 8
+
+    def load_data(self):
+        with open(self.dataset, 'r') as f:
+            self.datas = f.readlines()
+            self.num_data = len(self.datas)
+
+    def compute_state(self, index):
+        # state = self.state.copy()
+        self.state[-1] = np.zeros_like(self.state[-1])
+        [date_t, beam_type, user, ap] = self.datas[index].split('\t')
+        user = int(user)
+        ap = int(ap)
+        beam_type = int(beam_type)
+        i = user//10
+        j = user%10
+        t = 0
+        if ap == 0:
+            t = beam_type
+        else:
+            t = -beam_type
+        self.state[-1][i][j] = t
+        return self.state
+
+    def compute_reward(self, action):
+        assert type(action) == int
+        assert action >= 0 and action < self.action_space
+        # 功率暂时不用
+
+        # 频谱效率
+        axis2_sum = np.sum(self.state, axis= 2)
+        matrix_sum = np.sum(axis2_sum, axis=1)
+        if (matrix_sum)
+        t = 0.0
+        for i in range(self.action_space):
+            if matrix_sum[i] == 0:
+                t += 1/self.action_space
+            else:
+                t += 1/self.action_space * matrix_sum[i]/self.max_intervals
+        # 用户满意率，即action所在的信道的最大可用带宽的满足程度
+        b = 0.0
+        acquire = matrix_sum[-1]
+        supply = self.max_intervals - matrix_sum[action]
+        b = min(1.0, acquire/supply)
+        # 如果用户被拒绝了会有一个非常大的惩罚
+        if abs(b) < 1e-7 or supply <= 0:
+            b = -1.0
+
+        # 载波共用的惩罚
+        c:float = 0
+        # 暂时不动
+        return t + b
+
+
+def renew_environment():
+    pass
+def step(self, action):
+    self.state = self.compute_state(self.steps_before_termination)
+    reward = self.compute_reward(action)
+
+    # self.pre
+
+
+
+
+if __name__ == "__main__":
+    e = GEOEnv()
+    e.load_data()
+    print(len(e.datas))
+    state = e.compute_state(0)
+    print(state)
+    state[1][1][2] = 4
+    t = e.compute_reward(state, 1)
+    print(t)
+
+
diff --git a/GEO_parameters.py b/GEO_parameters.py
new file mode 100644
index 0000000..f6c94e6
--- /dev/null
+++ b/GEO_parameters.py
@@ -0,0 +1,13 @@
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+class GEO_Parameters:
+    Mc = 0
+    state_dim = 0
+    def __init__(self, Mc, state_dim):
+        # M_c代表的是载波的数量
+        # state_dim 代表的是数据的维度
+        # 32 * 32 代表的
+        GEO_Parameters.Mc = Mc
+        GEO_Parameters.state_dim = state_dim
diff --git a/dqn.py b/dqn.py
index 1ddab07..816c959 100644
--- a/dqn.py
+++ b/dqn.py
@@ -1,6 +1,15 @@
+from collections import deque
+
 import numpy as np
 import torch
 
+import GEO_parameters
+from Replay_memory import Memory, Transaction
+from model import Net
+import torch.nn.functional as f
+import matplotlib.pyplot as plt
+from GEO_parameters import GEO_Parameters
+
 
 def dev0(a, b):
     a = a.detach().numpy()
@@ -23,3 +32,149 @@ def hard_update(target, source):
 
 
 class DeepQNetwork:
+    def __init__(self, cap, n_actions, n_states, learning_rate, reward_decay, e_greedy, replace_target_iter, batch_size,e_greedy_increment=None):
+
+        self.memory = Memory(cap)
+        self.batch = None
+        self.state_batch = batch_size
+        self.action_batch = None
+        self.reward_batch = None
+        self.non_final_next_states = None
+        self.expected_state_action_values = None
+        self.state_action_values = None
+
+        self.n_states = n_states
+        self.n_actions = n_actions
+        self.lr = learning_rate
+        self.gamma = reward_decay
+        self.epsilon_max = e_greedy
+        self.replace_target_iter = replace_target_iter
+        self.batch_size = batch_size
+        self.epsilon_decay = e_greedy_increment
+        self.epsilon = 0 if e_greedy_increment is None else self.epsilon_max
+
+
+        self.learn_step_counter = 1
+
+        self.counter = GEO_parameters.state_dim
+        self.main_q_network = Net(n_states, n_actions) # 输入的nstates 应该是M_c+1
+        self.target_q_network = Net(n_states, n_actions)
+
+        self.optimizer = torch.optim.Adam(self.main_q_network.parameters(), self.lr)
+
+        self.cost_his = []
+
+        self.weight_dir = 'weight_DQN'
+
+    def replay(self):
+        if self.learn_step_counter % self.replace_target_iter == 0:
+            self.update_target_q_network()
+
+        if len(self.memory) <self.batch_size:
+            return
+
+        self.batch, self.state_batch, self.action_batch, self.reward_batch, self.non_final_next_states = self.make_mini_batch()
+        self.expected_state_action_values = self.get_expected_state_action_values()
+        self.update_main_q_network()
+        self.epsilon *= 0.9998
+        self.learn_step_counter += 1
+
+    def make_mini_batch(self):
+        transitions = self.memory.sample(self.batch_size)
+
+        batch = Transaction(*zip(*transitions))
+        state_batch = torch.cat(batch.state)
+        action_batch = torch.cat(batch.action)
+        reward_batch = torch.cat(batch.reward)
+        non_final_next_states = torch.cat([s for s in batch.next_state if s is not None])
+        return batch, state_batch, action_batch, reward_batch, non_final_next_states
+
+    def get_expected_state_action_values(self):
+        self.main_q_network.eval()
+        self.target_q_network.eval()
+
+        self.state_action_values = self.main_q_network
+
+        non_final_mask = torch.ByteTensor(tuple(map(lambda x : x is not None, self.batch.next_state))).bool()
+
+        next_state_values = torch.zeros(self.batch_size)
+
+        a_m = torch.zeros(self.batch_size).type(torch.LongTensor)
+
+        a_m[non_final_mask] = self.main_q_network(self.non_final_next_states).detach().max(1)[1]
+
+        a_m_non_final_next_states = a_m[non_final_mask].view(-1, 1)
+
+        next_state_values[non_final_mask] = self.target_q_network(
+            self.non_final_next_states
+        ).gather(1, a_m_non_final_next_states).detach().squeeze()
+
+        expected_state_action_values = self.reward_batch + self.gamma * next_state_values.view(-1, 1)
+        return expected_state_action_values
+
+    def decide_action(self, state):
+        # self.epsilon = 0.5 * (1 / (self.epsilon + 1))
+        if self.epsilon < np.random.uniform():
+            self.main_q_network.eval()
+            with torch.no_grad():
+                action = self.main_q_network(state).detach().max(1)[1]
+        else:
+            action = self.LongTenser([np.random.randint(0, self.n_actions)])
+        return action.view(1, 1)
+
+    def update_main_q_network(self):
+        self.main_q_network.train()
+
+        loss = f.mse_loss(self.state_action_values, self.expected_state_action_values, reduction = 'mean')
+        self.cost_his.append(loss.item())
+        self.optimizer.zero_grad()
+        loss.backward()
+        self.optimizer.step()
+
+    def update_target_q_network(self):
+        self.target_q_network.load_state_dict(self.main_q_network.state_dict())
+
+    def plot_cost(self):
+        plt.plot(np.arange(len(self.cost_his)), self.cost_his)
+        plt.xlabel('training step')
+        plt.ylabel('cost')
+        plt.show()
+
+    def binary(self, q):
+        self.counter = GEO_Parameters.state_dim - 1
+        t = torch.unsqueeze(q.max(1)[1], 1)
+        temp = torch.zeros_like(t)
+        temp_extend = temp.repeat(1, GEO_Parameters.state_dim)
+        n = temp
+        while torch.logical_not(torch.eq(n, t)):
+            t1 = t % 2
+            t2 = t >> 1
+            numColumns = temp_extend.shape[1]
+            new_tensor_left = temp_extend[:, :self.counter]
+            new_tensor_right = temp_extend[:, self.counter+1:numColumns]
+            temp_extend = torch.cat([new_tensor_left, new_tensor_right], 1)
+            self.counter -= 1
+
+        a1 = temp_extend
+        a1 = a1.float()
+        return a1
+
+    def load_weight_from_pkl(self, file_name):
+        checkpoint = torch.load(file_name)
+        self.main_q_network.load_state_dict(checkpoint['main_q_network_state_dict'])
+        self.target_q_network.load_state_dict(checkpoint['target_q_network_state_dict'])
+        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        self.cost_his = checkpoint['loss']
+
+    def save_weight_to_pkl(self, file_name):
+        torch.save({
+            # 'epoch': epochs,
+            'main_q_network_state_dict': self.main_q_network.state_dict(),
+            'target_q_network_state_dict': self.target_q_network.state_dict(),
+            'optimizer_state_dict': self.optimizer.state_dict(),
+            'loss': self.cost_his,
+        }, file_name)
+
+    def update_replay_buffer(self, state, action, reward, next_state):
+        self.memory.push(state, action, reward, next_state)
+
diff --git a/drl_framework.py b/drl_framework.py
new file mode 100644
index 0000000..968e922
--- /dev/null
+++ b/drl_framework.py
@@ -0,0 +1,29 @@
+from dqn import DeepQNetwork
+from GEOEnv import GEOEnv
+import matplotlib.pyplot as pyplot
+
+class DRL:
+    def __init__(self, num_episodes):
+        self.env = GEOEnv()
+
+        self.agent = DeepQNetwork()
+        self.steps = 0
+
+
+    def interact(self, state):
+        action = self.agent.decide_action(state)
+        next_state, reward, done = self.env.step(action)
+        self.agent.update_replay_buffer(state, action, reward, next_state)
+
+        state = next_state
+
+        self.steps += 1
+        # update epsilon
+
+        return state, reward, done
+
+    def episodic_learn(self):
+        for episode in range(self.agent.max_episodes):
+            state = self.env.reset()
+            done = False
+            self.steps = 0
diff --git a/model.py b/model.py
index b116ab7..d90d2c3 100644
--- a/model.py
+++ b/model.py
@@ -10,6 +10,11 @@ M = 16
 
 class Net(nn.Module):
     def __init__(self, n_input_channels, n_actions):
+        ## input channels 是数据的输入的维度
+        ## 每个状态是一个
+        ## N * N * (M_c + 1)的tensor
+        ## 其中 N * N 代表的是用户的数量 如果有1000个用户 大概为32
+        ## 第M_c个matrix代表的是当前用户的申请或是离开
         super(Net, self).__init__()
         self.conv1 = nn.Conv2d(in_channels=n_input_channels, out_channels=16, kernel_size=5, stride=1, padding=0)
         self.conv2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, stride=1, padding=0)