11

.idea/.gitignore

@@ -0,0 +1,8 @@
+# 默认忽略的文件
+/shelf/
+/workspace.xml
+# 基于编辑器的 HTTP 客户端请求
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/deployment.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" remoteFilesAllowedToDisappearOnAutoupload="false">
+    <serverData>
+      <paths name="dsab">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+    </serverData>
+  </component>
+</project>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.7" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="py310" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/thesis-code.iml" filepath="$PROJECT_DIR$/.idea/thesis-code.iml" />
+    </modules>
+  </component>
+</project>

.idea/thesis-code.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="py310" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

Replay_memory.py

@@ -0,0 +1,24 @@
+import random
+from collections import namedtuple
+
+Transaction = namedtuple('Transaction', ('state', 'action', 'reward', 'next_state'))
+
+
+class Memory(object):
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self.memory = []
+        self.index = 0
+
+    def push(self, state, action, reward, next_state):
+        if len(self.memory) < self.capacity:
+            self.memory.append(None)
+        self.memory[self.index] = (state, action, reward, next_state)
+        self.index = (self.index + 1) % self.capacity
+
+    def sample(self, batch_size):
+        assert len(self.memory) >= self.capacity
+        return random.sample(self.memory, batch_size)
+
+    def __len__(self):
+        return len(self.memory)

dqn.py

@@ -0,0 +1,25 @@
+import numpy as np
+import torch
+
+
+def dev0(a, b):
+    a = a.detach().numpy()
+    b = b.detach().numpy()
+    with np.errstate(divide='ignore', invalid='ignore'):
+        c = np.true_divide(a, b)
+        c[~ np.isfinite(c)] = 0
+    c = torch.from_numpy(c)
+    return c
+
+
+def soft_update(target, source, tau):
+    for target_param, param in zip(target.parameters(), source.parameters()):
+        target_param.data.copy_(target_param.data * (1.0 - tau) + param.data * tau)
+
+
+def hard_update(target, source):
+    for target_param, param in zip(target.parameters(), source.parameters()):
+        target_param.data.copy_(param.data)
+
+
+class DeepQNetwork:

encode.py

@@ -0,0 +1 @@
+

model.py

@@ -0,0 +1,46 @@
+import torch
+import torch.nn as nn
+
+N_VEL = 2
+N_SUB = 2
+state_dim = N_VEL * N_SUB
+num_actions = 2 ** (N_SUB * N_VEL)
+M = 16
+
+
+class Net(nn.Module):
+    def __init__(self, n_input_channels, n_actions):
+        super(Net, self).__init__()
+        self.conv1 = nn.Conv2d(in_channels=n_input_channels, out_channels=16, kernel_size=5, stride=1, padding=0)
+        self.conv2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, stride=1, padding=0)
+        self.fc1 = nn.Linear(in_features=4 * 4 * 32, out_features=512)
+        self.fc2 = nn.Linear(in_features=512, out_features=512)
+        self.fc3 = nn.Linear(in_features=512, out_features=n_actions)
+        # nn.init.normal_(self.fc1.weight, 0., 0.1)
+        # nn.init.constant_(self.fc1.bias, 0.1)
+        # self.fc2 = nn.Linear(500, 300)
+        # nn.init.normal_(self.fc2.weight, 0., 0.1)
+        # nn.init.constant_(self.fc2.bias, 0.1)
+        # self.fc3 = nn.Linear(300, 250)
+        # nn.init.normal_(self.fc3.weight, 0., 0.1)
+        # nn.init.constant_(self.fc3.bias, 0.1)
+        # self.fc4 = nn.Linear(250, n_actions)
+        # nn.init.normal_(self.fc4.weight, 0., 0.1)
+        # nn.init.constant_(self.fc4.bias, 0.1)
+
+    def forward(self, x):
+        x = torch.sigmoid(self.conv1(x))
+        x = torch.sigmoid(self.conv2(x))
+        x = x.view(x.size(0), -1)
+        x = torch.sigmoid(self.fc1(x))
+        x = torch.sigmoid(self.fc2(x))
+        output = self.fc3(x)
+        return output
+
+
+if __name__ == "__main__":
+    input_channels = 10 * 10 * 16
+    num_actions = 16
+    q_network = Net(input_channels, num_actions)
+
+    print(q_network)