添加 A2C/QAC 算法实现及训练结果

- 新增 RL_Algothrithms 模块，包含 A2C、QAC 智能体
- 添加 SAC 章节笔记和 C10 笔记
- 上传训练结果图片
- 完善 README 与 .gitignore

RL_Algothrithms/agents/qac.py

@@ -0,0 +1,59 @@
+import torch
+import torch.optim as optim
+import torch.nn.functional as F
+import torch.distributions as distributions
+from networks import Actor, QCritic
+
+class QACAgent:
+    def __init__(self, state_dim, action_dim, device, actor_lr=0.001, critic_lr=0.002, gamma=0.99):
+        # 接收设备参数，确保网络挂载在 GPU 或 CPU 上
+        self.device = device
+        self.gamma = gamma
+        
+        # 实例化网络并移动到指定设备
+        self.actor = Actor(state_dim, action_dim).to(self.device)
+        self.critic = QCritic(state_dim, action_dim).to(self.device)
+        
+        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr)
+        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_lr)
+
+    def select_action(self, state):
+        # 推理时禁用梯度图计算，加快速度
+        with torch.no_grad():
+            state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+            action_probs = self.actor(state_tensor)
+            action = distributions.Categorical(action_probs).sample().item()
+        return action
+
+    def update(self, state, action, reward, next_state, next_action, done):
+        # 将数据转换为张量并送入 GPU
+        state = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+        next_state = torch.FloatTensor(next_state).unsqueeze(0).to(self.device)
+        reward = torch.FloatTensor([reward]).unsqueeze(0).to(self.device)
+        
+        # --- Critic 更新 ---
+        q_values = self.critic(state)
+        current_q = q_values[0, action]
+        
+        next_q_values = self.critic(next_state).detach()
+        next_q = next_q_values[0, next_action]
+        
+        # 计算 TD 目标
+        td_target = reward + self.gamma * next_q * (1 - int(done))
+        critic_loss = F.mse_loss(current_q, td_target.squeeze())
+        
+        self.critic_optimizer.zero_grad()
+        critic_loss.backward()
+        self.critic_optimizer.step()
+        
+        # --- Actor 更新 ---
+        action_probs = self.actor(state)
+        dist = distributions.Categorical(action_probs)
+        log_prob = dist.log_prob(torch.tensor([action]).to(self.device))
+        
+        # Actor 梯度上升目标：ln(pi) * Q(s, a)
+        actor_loss = -(log_prob * current_q.detach())
+        
+        self.actor_optimizer.zero_grad()
+        actor_loss.backward()
+        self.actor_optimizer.step()