first commit

utils/__init__.py

@@ -0,0 +1,110 @@
+import torch
+import torch.nn.functional as F
+import torch.optim as optim
+from models.networks import Actor, Critic
+import copy
+
+class SAC(object):
+    def __init__(self, state_dim, action_dim, max_action, config):
+        """
+        初始化 Soft Actor-Critic 算法
+        """
+        # 设备配置 (CPU 或 GPU)
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        
+        # 从 config 字典中加载超参数
+        self.gamma = config.get('gamma', 0.99)        # 折扣因子
+        self.tau = config.get('tau', 0.005)           # 目标网络软更新系数 (论文公式 9 下方)
+        self.alpha = config.get('alpha', 0.2)         # 熵的温度参数 (控制探索的随机性)
+        
+        # 1. 实例化策略网络 (Actor)
+        self.actor = Actor(state_dim, action_dim, max_action).to(self.device)
+        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=config.get('lr', 3e-4))
+        
+        # 2. 实例化价值网络 (Critic) - 内部已经包含了 Q1 和 Q2
+        self.critic = Critic(state_dim, action_dim).to(self.device)
+        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=config.get('lr', 3e-4))
+        
+        # 3. 实例化目标价值网络 (Target Critic)
+        # 用 copy.deepcopy 完美复制一份初始参数，并冻结其梯度计算
+        self.critic_target = copy.deepcopy(self.critic)
+        for param in self.critic_target.parameters():
+            param.requires_grad = False
+
+    def select_action(self, state, evaluate=False):
+        """
+        与环境交互时使用的动作选择函数
+        """
+        # 将输入的状态转换为 PyTorch Tensor
+        state = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+        
+        # 论文技巧：评估（测试）时使用均值动作，训练时使用采样动作
+        with torch.no_grad():
+            if evaluate:
+                _, _, action = self.actor.sample(state) # 第三个返回值是均值
+            else:
+                action, _, _ = self.actor.sample(state) # 第一个返回值是加了噪声的采样值
+                
+        # 转换回 numpy 数组，送给 Gym 环境执行
+        return action.cpu().data.numpy().flatten()
+
+    def update(self, replay_buffer, batch_size):
+        """
+        算法的核心心跳：从经验池采样并更新神经网络参数
+        """
+        # 从 Replay Buffer 中随机抽取一个 Batch 的数据
+        state, action, reward, next_state, not_done = replay_buffer.sample(batch_size)
+
+        # ================================================================= #
+        #                           1. 更新 Critic                          #
+        # ================================================================= #
+        with torch.no_grad():
+            # 拿到下一个状态的动作和其对应的对数概率 (用于计算熵)
+            next_action, next_log_prob, _ = self.actor.sample(next_state)
+            
+            # 使用目标网络计算下一个状态的 Q 值 (Q1 和 Q2)
+            target_Q1, target_Q2 = self.critic_target(next_state, next_action)
+            
+            # 【核心对抗高估】：取两个 Q 值的最小值
+            target_Q = torch.min(target_Q1, target_Q2)
+            
+            # 【软贝尔曼方程】：目标 Q 值 = 奖励 + gamma * (目标 Q - alpha * 熵)
+            # 注意这里加上了 -self.alpha * next_log_prob，这就是论文中“把熵当做奖励”的体现
+            target_Q = reward + not_done * self.gamma * (target_Q - self.alpha * next_log_prob)
+
+        # 获取当前状态和动作对应的 Q 值预测
+        current_Q1, current_Q2 = self.critic(state, action)
+
+        # 计算 Critic 的损失 (均方误差 MSE)
+        critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss(current_Q2, target_Q)
+
+        # 优化 Critic 网络
+        self.critic_optimizer.zero_grad()
+        critic_loss.backward()
+        self.critic_optimizer.step()
+
+        # ================================================================= #
+        #                           2. 更新 Actor                           #
+        # ================================================================= #
+        # 让当前 Actor 对**当前状态**重新采样一个动作 (注意：不能用 Buffer 里的旧动作)
+        pi_action, log_prob, _ = self.actor.sample(state)
+        
+        # 拿到更新后的 Critic 对这个新动作的打分
+        q1_pi, q2_pi = self.critic(state, pi_action)
+        min_q_pi = torch.min(q1_pi, q2_pi)
+
+        # 计算 Actor 的损失：最小化 (alpha * log_prob - min_Q)
+        # 等价于最大化 (min_Q - alpha * log_prob) -> 既要 Q 值大，又要熵大（分布广）
+        actor_loss = (self.alpha * log_prob - min_q_pi).mean()
+
+        # 优化 Actor 网络
+        self.actor_optimizer.zero_grad()
+        actor_loss.backward()
+        self.actor_optimizer.step()
+
+        # ================================================================= #
+        #                    3. 软更新 Target Critic                        #
+        # ================================================================= #
+        # 使用 EMA (指数移动平均) 缓慢将前线 Critic 的参数移交给 Target Critic
+        for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)

utils/logger.py

@@ -0,0 +1,62 @@
+# ==============================================================================
+# Author: Hongru Liu
+# Affiliation: School of Power and Energy, Northwestern Polytechnical University
+# Version: 1.0
+# Contact: hongruliu@mail.nwpu.edu.cn
+# ==============================================================================
+
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+class Logger(object):
+    def __init__(self):
+        """
+        初始化日志记录器，用于暂存训练过程中的各项指标
+        """
+        self.episode_rewards = []
+        
+    def record(self, reward):
+        """
+        记录每个 Episode 的总奖励
+        """
+        self.episode_rewards.append(reward)
+        
+    def plot_learning_curve(self, save_dir="."):
+        """
+        绘制并保存学习曲线 (Learning Curve)
+        """
+        # 确保保存目录存在
+        os.makedirs(save_dir, exist_ok=True)
+        
+        # 创建画布，严格设置白色背景
+        fig, ax = plt.subplots(figsize=(10, 6), facecolor='white')
+        ax.set_facecolor('white')
+        
+        # 绘制奖励曲线，使用加粗线条以满足论文发表的视觉要求
+        ax.plot(self.episode_rewards, linewidth=2.5, color='#1f77b4', label='Episode Reward')
+        
+        # 计算并绘制 10 个 Episode 的滑动平均线，让趋势更清晰
+        if len(self.episode_rewards) >= 10:
+            moving_avg = np.convolve(self.episode_rewards, np.ones(10)/10, mode='valid')
+            ax.plot(range(9, len(self.episode_rewards)), moving_avg, 
+                    linewidth=2.5, color='#ff7f0e', label='10-Episode Moving Average')
+
+        # 设置全英文的坐标轴标签和图例，调整字体大小
+        ax.set_xlabel('Episodes', fontsize=14, fontweight='bold')
+        ax.set_ylabel('Total Reward', fontsize=14, fontweight='bold')
+        ax.set_title('Training Learning Curve (Pendulum-v1)', fontsize=16, fontweight='bold')
+        
+        # 设置刻度字体大小
+        ax.tick_params(axis='both', which='major', labelsize=12)
+        
+        # 增加网格线并设置图例
+        ax.grid(True, linestyle='--', alpha=0.7)
+        ax.legend(fontsize=12, loc='lower right')
+        
+        # 紧凑布局并保存
+        plt.tight_layout()
+        save_path = os.path.join(save_dir, "learning_curve.png")
+        plt.savefig(save_path, dpi=300, facecolor=fig.get_facecolor(), edgecolor='none')
+        plt.close()
+        print(f"[*] Learning curve saved to: {save_path}")

utils/replay_buffer.py

@@ -0,0 +1,59 @@
+import numpy as np
+import torch
+
+class ReplayBuffer(object):
+    def __init__(self, state_dim, action_dim, max_size=int(1e6)):
+        """
+        初始化经验回放池
+        使用预分配的 Numpy 数组来提升存储和采样效率
+        """
+        self.max_size = max_size
+        self.ptr = 0     # 当前写入的指针位置
+        self.size = 0    # 当前池子里的有效数据量
+
+        # 预先分配内存，避免动态扩张带来性能开销
+        self.state = np.zeros((max_size, state_dim), dtype=np.float32)
+        self.action = np.zeros((max_size, action_dim), dtype=np.float32)
+        self.reward = np.zeros((max_size, 1), dtype=np.float32)
+        self.next_state = np.zeros((max_size, state_dim), dtype=np.float32)
+        
+        # 记录该状态是否是回合的结束 (1.0 表示结束，0.0 表示未结束)
+        self.not_done = np.zeros((max_size, 1), dtype=np.float32)
+
+        # 自动检测 GPU
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    def add(self, state, action, reward, next_state, done):
+        """
+        向回放池中添加一条新的转移数据 (Transition)
+        """
+        # 将数据写入指针当前所在的位置
+        self.state[self.ptr] = state
+        self.action[self.ptr] = action
+        self.reward[self.ptr] = reward
+        self.next_state[self.ptr] = next_state
+        
+        # 我们存储 1 - done，这样在贝尔曼方程更新时直接相乘即可：
+        # Q_target = r + gamma * V * not_done
+        self.not_done[self.ptr] = 1. - done
+
+        # 移动指针，如果达到了最大容量，就回到开头覆盖最老的数据（环形结构）
+        self.ptr = (self.ptr + 1) % self.max_size
+        # 更新当前有效数据量
+        self.size = min(self.size + 1, self.max_size)
+
+    def sample(self, batch_size):
+        """
+        随机采样一个 batch 的数据，并直接转换为 PyTorch Tensor 放到 GPU/CPU 上
+        """
+        # 在 0 到当前有效数据量之间，随机生成 batch_size 个索引
+        ind = np.random.randint(0, self.size, size=batch_size)
+
+        # 提取数据并转为 Tensor
+        return (
+            torch.FloatTensor(self.state[ind]).to(self.device),
+            torch.FloatTensor(self.action[ind]).to(self.device),
+            torch.FloatTensor(self.reward[ind]).to(self.device),
+            torch.FloatTensor(self.next_state[ind]).to(self.device),
+            torch.FloatTensor(self.not_done[ind]).to(self.device)
+        )