2026-02-28 00:18:43 +08:00
|
|
|
|
{
|
|
|
|
|
|
"cells": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "markdown",
|
|
|
|
|
|
"id": "e0ac366f",
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"# 第 5 章：蒙特卡洛方法 (Monte Carlo Methods)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"## 1. 什么是蒙特卡洛 (MC)？\n",
|
|
|
|
|
|
"当智能体不知道环境的运作规律（无模型，Model-Free）时，它只能通过与环境真实交互来学习。\n",
|
|
|
|
|
|
"蒙特卡洛方法的核心思想是：**“大数定律”**。既然我算不出某个状态的理论预期价值，那我就从这个状态出发，亲自跑几十次、几百次完整的**回合（Episode）**，然后把实际得到的**回报（Return, $G_t$）取平均值**。跑的次数越多，平均值就越接近真实的价值。\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"## 2. 核心特征\n",
|
|
|
|
|
|
"* **必须是分步的（Episodic）**：蒙特卡洛必须等一个完整的回合（比如一局游戏）彻底结束后，才能从后往前计算总回报并更新价值。\n",
|
|
|
|
|
|
"* **计算动作价值 $q(s,a)$**：因为没有模型，光知道状态价值 $v(s)$ 没用了（你不知道选哪个动作能进入好状态）。因此，MC 直接估计**动作价值 $q(s,a)$**。\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"## 3. 探索与利用 ($\\epsilon$-Greedy 策略)\n",
|
|
|
|
|
|
"既然要靠“试错”来积累经验，智能体就绝不能总是死盯着当前看起来最好的动作（利用 Exploitation），它必须保留一定的概率去尝试其他动作（探索 Exploration）。\n",
|
|
|
|
|
|
"**$\\epsilon$-贪心（$\\epsilon$-Greedy）策略**：\n",
|
|
|
|
|
|
"* 以 $1 - \\epsilon$ 的概率选择当前 Q 值最大的最佳动作。\n",
|
|
|
|
|
|
"* 以 $\\epsilon$ 的概率在所有动作中**随机盲选**（这就是探索！）。"
|
|
|
|
|
|
]
|
|
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"execution_count": 1,
|
|
|
|
|
|
"id": "11448339",
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"outputs": [],
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"import numpy as np\n",
|
|
|
|
|
|
"import random\n",
|
|
|
|
|
|
"from collections import defaultdict\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 1. 简单的 1D 走廊环境 (黑盒)\n",
|
|
|
|
|
|
"# 状态: 0, 1, 2, 3 (3 是目标宝箱)\n",
|
|
|
|
|
|
"# 动作: 0 (向左), 1 (向右)\n",
|
|
|
|
|
|
"def step(state, action):\n",
|
|
|
|
|
|
" if state == 3: # 已经在终点\n",
|
|
|
|
|
|
" return 3, 0, True \n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" if action == 1: # 向右走\n",
|
|
|
|
|
|
" next_state = state + 1\n",
|
|
|
|
|
|
" else: # 向左走\n",
|
|
|
|
|
|
" next_state = max(0, state - 1)\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" reward = 10 if next_state == 3 else -1\n",
|
|
|
|
|
|
" done = (next_state == 3) # 是否结束回合\n",
|
|
|
|
|
|
" return next_state, reward, done\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 2. 定义 epsilon-greedy 策略\n",
|
|
|
|
|
|
"def epsilon_greedy_policy(state, Q, epsilon, n_actions=2):\n",
|
|
|
|
|
|
" # 以 epsilon 的概率随机探索\n",
|
|
|
|
|
|
" if random.uniform(0, 1) < epsilon:\n",
|
|
|
|
|
|
" return random.choice(range(n_actions))\n",
|
|
|
|
|
|
" # 以 1 - epsilon 的概率贪心利用 (选择 Q 值最大的动作)\n",
|
|
|
|
|
|
" else:\n",
|
|
|
|
|
|
" # 如果 Q 值全是 0，也会默认选第一个，所以用 argmax\n",
|
|
|
|
|
|
" return np.argmax(Q[state])"
|
|
|
|
|
|
]
|
|
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"execution_count": 3,
|
|
|
|
|
|
"id": "7ecc277d",
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
|
|
|
|
|
"=== 开始蒙特卡洛控制 (Monte Carlo Control) ===\n",
|
|
|
|
|
|
"完成第 100 个回合训练...\n",
|
|
|
|
|
|
"完成第 200 个回合训练...\n",
|
|
|
|
|
|
"完成第 300 个回合训练...\n",
|
|
|
|
|
|
"完成第 400 个回合训练...\n",
|
|
|
|
|
|
"完成第 500 个回合训练...\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"--- 训练结束！揭晓学到的 Q 表 ---\n",
|
|
|
|
|
|
"状态 0: 向左 Q=3.44, 向右 Q=5.38 -> 最优动作: 向右\n",
|
|
|
|
|
|
"状态 1: 向左 Q=3.03, 向右 Q=7.49 -> 最优动作: 向右\n",
|
|
|
|
|
|
"状态 2: 向左 Q=5.10, 向右 Q=10.00 -> 最优动作: 向右\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"结论：即使不知道环境具体规则，智能体仅凭不断试错取平均，也学会了一直向右走才是通关秘籍！\n"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"print(\"=== 开始蒙特卡洛控制 (Monte Carlo Control) ===\")\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 初始化 Q 表 (状态数目为4，动作为2)\n",
|
|
|
|
|
|
"# 使用 defaultdict 方便处理没见过的状态\n",
|
|
|
|
|
|
"Q = defaultdict(lambda: np.zeros(2))\n",
|
|
|
|
|
|
"# 用于记录每个 (状态, 动作) 组合被访问了多少次，以及获得的总回报\n",
|
|
|
|
|
|
"returns_sum = defaultdict(float)\n",
|
|
|
|
|
|
"returns_count = defaultdict(float)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"num_episodes = 500\n",
|
|
|
|
|
|
"gamma = 0.9\n",
|
|
|
|
|
|
"epsilon = 0.2\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"for i in range(num_episodes):\n",
|
|
|
|
|
|
" # --- 1. 生成一个完整的回合 (Episode) ---\n",
|
|
|
|
|
|
" episode = []\n",
|
|
|
|
|
|
" state = 0 # 每次都从起点开始\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # 智能体开始在黑盒里凭感觉走，直到碰壁或找到宝箱\n",
|
|
|
|
|
|
" while True:\n",
|
|
|
|
|
|
" action = epsilon_greedy_policy(state, Q, epsilon)\n",
|
|
|
|
|
|
" next_state, reward, done = step(state, action)\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # 记录下这一步的“经验”: (状态, 动作, 奖励)\n",
|
|
|
|
|
|
" episode.append((state, action, reward))\n",
|
|
|
|
|
|
" state = next_state\n",
|
|
|
|
|
|
" if done:\n",
|
|
|
|
|
|
" break\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # --- 2. 回合结束后，从后往前算回报并更新 Q 表 ---\n",
|
|
|
|
|
|
" G = 0.0 # G 代表累计回报 Return\n",
|
|
|
|
|
|
" # 从轨迹的最后一步倒着往前算\n",
|
|
|
|
|
|
" for t in reversed(range(len(episode))):\n",
|
|
|
|
|
|
" state, action, reward = episode[t]\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # 计算折扣回报\n",
|
|
|
|
|
|
" G = gamma * G + reward\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # First-Visit MC (初次访问蒙特卡洛): \n",
|
|
|
|
|
|
" # 只在回合中首次遇到这个 (状态,动作) 时才更新\n",
|
|
|
|
|
|
" state_action_pairs_before_t = [(x[0], x[1]) for x in episode[:t]]\n",
|
|
|
|
|
|
" if (state, action) not in state_action_pairs_before_t:\n",
|
|
|
|
|
|
" # 记录总回报并计数\n",
|
|
|
|
|
|
" returns_sum[(state, action)] += G\n",
|
|
|
|
|
|
" returns_count[(state, action)] += 1.0\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # 平均值法更新 Q 表: Q = sum(G) / count\n",
|
|
|
|
|
|
" Q[state][action] = returns_sum[(state, action)] / returns_count[(state, action)]\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 打印部分训练过程\n",
|
|
|
|
|
|
" if (i + 1) % 100 == 0:\n",
|
|
|
|
|
|
" print(f\"完成第 {i + 1} 个回合训练...\")\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"print(\"\\n--- 训练结束！揭晓学到的 Q 表 ---\")\n",
|
|
|
|
|
|
"for s in range(3):\n",
|
|
|
|
|
|
" print(f\"状态 {s}: 向左 Q={Q[s][0]:.2f}, 向右 Q={Q[s][1]:.2f} -> 最优动作: {'向右' if np.argmax(Q[s])==1 else '向左'}\")\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"print(\"\\n结论：即使不知道环境具体规则，智能体仅凭不断试错取平均，也学会了一直向右走才是通关秘籍！\")"
|
|
|
|
|
|
]
|
|
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "markdown",
|
|
|
|
|
|
"id": "8e92690e",
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"加一个小测试，引入$\\epsilon$ 衰减（Epsilon Decay）的机制，并与之前的方法进行对比"
|
|
|
|
|
|
]
|
|
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
2026-02-28 15:27:59 +08:00
|
|
|
|
"execution_count": null,
|
2026-02-28 00:18:43 +08:00
|
|
|
|
"id": "142b7542",
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"data": {
|
|
|
|
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA0gAAAIhCAYAAAB0a5r9AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXd8VFXax393eia99x4ILUBCB2kiINg76iLYlbW7rr3uuq66r2tZFSsWFLGiAkoTkN4Sek/vfSZ9MuW8f5x77/RkkkySmeR8P5/JzNx6cu+de89znuf5PRwhhIDBYDAYDAaDwWAwGJD0dwMYDAaDwWAwGAwGw1NgBhKDwWAwGAwGg8Fg8DADicFgMBgMBoPBYDB4mIHEYDAYDAaDwWAwGDzMQGIwGAwGg8FgMBgMHmYgMRgMBoPBYDAYDAYPM5AYDAaDwWAwGAwGg4cZSAwGg8FgMBgMBoPBwwwkBoPBYDAYDAaDweBhBhKD0Y989tln4DgOHMdh27ZtdvMJIUhLSwPHcZg1a1avtmX37t144YUXoNFoemX7eXl5uO+++zB06FD4+PhArVZj5MiReOaZZ1BaWurWfb3wwgvgOM6t2wSAo0eP4tZbb0VycjJUKhX8/PyQlZWF1157DXV1dW7fn0BZWRleeOEFHD58uFe2n5SUhKVLl/ZoG5WVlXjiiSeQkZEBPz8/qFQqDBkyBA8++CDOnTvnnoZ2k94+fra89957+Oyzz/pkX64g3GM4joNUKkVwcDDGjBmDu+++G3v37u3v5nWZpUuXIikpqd/2bXk8bV+9xaxZs+yeARzH4YUXXui1fTIYgxlZfzeAwWAA/v7++OSTT+wegNu3b0dubi78/f17vQ27d+/Giy++iKVLlyIoKMit2167di0WLVqEsLAw3HfffcjMzATHcTh27Bg+/fRTrFu3Djk5OW7dp7v56KOPsGzZMqSnp+Oxxx7DiBEjoNfrcfDgQSxfvhx79uzBTz/91Cv7Lisrw4svvoikpCSMHTu2V/bRE/bv349LL70UhBDcd999mDJlChQKBc6cOYOVK1di4sSJqK+v77f29fXxe++99xAWFtZjo9OdXHvttXj00UdBCEFDQwOOHz+OL774Ah9++CEeeOABvPXWW/3dRJd59tln8eCDD/bb/n18fPDHH3/06T7fe++9Pt0fgzHYYQYSg+EB3HDDDfjqq6/w7rvvIiAgQJz+ySefYMqUKWhoaOjH1vWM/Px8LFq0CEOHDsXWrVsRGBgozrvwwgvxwAMPuM2waGlpgVqtdsu2LNmzZw/uvfdezJ07F2vWrIFSqRTnzZ07F48++ih+//13t++3u/TWcXBEQ0MDrrjiCqhUKuzevRtxcXHivFmzZuHuu+/G999/3ydtYTgnMjISkydPFr/Pnz8fDz30EO666y68/fbbGDZsGO69995+bKHrpKam9uv+JRKJ1bHsC0aMGNGn+2MwBjssxI7B8ABuvPFGAMCqVavEaVqtFj/88ANuu+02h+vU1dVh2bJliI2NhUKhQEpKCp5++mnodDqr5TiOw3333Ycvv/wSw4cPh1qtxpgxY7B27VpxmRdeeAGPPfYYACA5Odlh2N/q1asxZcoU+Pr6ws/PD/Pnz3fJ6/PGG2+gubkZ7733npVxZNm+q6++Wvy+adMmXHHFFYiLi4NKpUJaWhruvvtu1NTUWK0nhNFlZ2fj2muvRXBwcIcdJ5PJhNdeew3Dhg2DUqlEREQEbrnlFpSUlHT6P/zrX/8Cx3H48MMPrYwjAYVCgcsvv7zL+5o1axZGjRqFAwcOYPr06VCr1UhJScG///1vmEwmAMC2bdswYcIEAMCtt94qnhshtGbp0qXw8/PDsWPHMG/ePPj7+2POnDkAXL9GesJHH32EiooKvPbaa1bGkSXXXnut1fdffvkFU6ZMgVqthr+/P+bOnYs9e/ZYLSOc3xMnTuDGG29EYGAgIiMjcdttt0Gr1Vot+91332HSpEkIDAwUj6Hwu+ns+B08eBCLFi1CUlISfHx8kJSUhBtvvBGFhYVW+xDCYbdu3Yp7770XYWFhCA0NxdVXX42ysjJxuaSkJJw4cQLbt28X99VROFhmZiamT59uN91oNCI2Ntbqt/H+++9jzJgx8PPzg7+/P4YNG4annnrK6bY7QyqV4n//+x/CwsLw+uuvW81raGjA3/72NyQnJ0OhUCA2NhYPPfQQmpubrZYzmUx45513MHbsWPj4+CAoKAiTJ0/GL7/8Ii6zevVqzJs3D9HR0fDx8cHw4cPxxBNPWG3ryy+/BMdxdtcBALz00kuQy+XicXYUYufKfU7g559/xujRo6FUKpGSkoK33nrL7WG527ZtA8dxWLlyJR555BFERUXBx8cHM2fOtLtv5uXlYdGiRYiJiYFSqURkZCTmzJljFRLqKMTOEcePH8cVV1yB4OBgqFQqjB07Fp9//rnDtq1atQpPP/00YmJiEBAQgIsuughnzpxxx7/PYHg/hMFg9BsrVqwgAMiBAwfI4sWLycSJE8V577//PvH19SUNDQ1k5MiRZObMmeK81tZWMnr0aOLr60v+85//kI0bN5Jnn32WyGQysnDhQqt9ACBJSUlk4sSJ5NtvvyXr168ns2bNIjKZjOTm5hJCCCkuLib3338/AUB+/PFHsmfPHrJnzx6i1WoJIYS8/PLLhOM4ctttt5G1a9eSH3/8kUyZMoX4+vqSEydOdPg/Dh06lERGRrp8TN5//33yyiuvkF9++YVs376dfP7552TMmDEkPT2dtLe3i8s9//zzBABJTEwkjz/+ONm0aRNZs2aN1TxL7rrrLgKA3HfffeT3338ny5cvJ+Hh4SQ+Pp5UV1c7bY/BYCBqtZpMmjTJ5f/B1X3NnDmThIaGkiFDhpDly5eTTZs2kWXLlhEA5PPPPyeEEKLVasXr5JlnnhHPTXFxMSGEkCVLlhC5XE6SkpLIK6+8QrZs2UI2bNjQpWskMTGRLFmyxGrazJkz7Y6hI+bNm0ekUilpampy6dh89dVXBACZN28eWbNmDVm9ejUZN24cUSgUZMeOHeJywjlMT08nzz33HNm0aRN54403iFKpJLfeequ43O7duwnHcWTRokVk/fr15I8//iArVqwgixcvdun4fffdd+S5554jP/30E9m+fTv55ptvyMyZM0l4eLjVuRK2kZKSQu6//36yYcMG8vHHH5Pg4GAye/Zscbns7GySkpJCMjMzxX1lZ2c7PR5vvfUWAUDOnj1rNX39+vUEAPnll18IIYSsWrWKACD3338/2bhxI9m8eTNZvnw5eeCBBzo95gDIX//6V6fzFy1aRACIx6S5uZmMHTuWhIWFkTfeeINs3ryZvPXWWyQwMJBceOGFxGQyiesuXryYcBxH7rjjDvLzzz+T3377jbz88svkrbfeEpf5xz/+Qf773/+SdevWkW3btpHly5eT5ORkq+Om0+lIVFQUufnmm63aptfrSUxMDLnuuuvEaUuWLCGJiYl2/2Nn9zlCCPntt9+IRCIhs2bNIj/99BP57rvvyKRJk0hSUpJL1/uSJUuIr68v0ev1di+j0Sgut3XrVgKAxMfHkyuuuIL8+uuvZOXKlSQtLY0EBARYtSk9PZ2kpaWRL7/8kmzfvp388MMP5NFHHyVbt24Vl5k5c6bVM0D4n59//nnx++nTp4m/vz9JTU0lX3zxBVm3bh258cYbCQDy6quv2rUtKSmJ3HzzzWTdunVk1apVJCEhgQwZMoQYDIZOjwODMdBhBhKD0Y9YGkjCQ+v48eOEEEImTJhAli5dSgghdgbS8uXLCQDy7bffWm3v1VdfJQDIxo0bxWkASGRkJGloaBCnVVRUEIlEQl555RVx2uuvv04AkPz8fKttFhUVEZlMRu6//36r6Y2NjSQqKopcf/31Hf6PKpWKTJ48ufOD4QCTyUT0ej0pLCwkAMjPP/8szhM60M8995zderYG0qlTpwgAsmzZMqvl9u3bRwCQp556ymkbKioqCACyaNEil9rclX0JRsi+ffuslh0xYgSZP3+++P3AgQMEAFmxYoXd/pYsWUIAkE8//dRqeleuEUc
|
|
|
|
|
|
"text/plain": [
|
|
|
|
|
|
"<Figure size 1000x600 with 1 Axes>"
|
|
|
|
|
|
]
|
|
|
|
|
|
},
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"output_type": "display_data"
|
2026-02-28 15:27:59 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"ename": "",
|
|
|
|
|
|
"evalue": "",
|
|
|
|
|
|
"output_type": "error",
|
|
|
|
|
|
"traceback": [
|
|
|
|
|
|
"\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
|
|
|
|
|
|
"\u001b[1;31m请查看单元格中的代码，以确定故障的可能原因。\n",
|
|
|
|
|
|
"\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
|
|
|
|
|
|
"\u001b[1;31m有关更多详细信息，请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
|
|
|
|
|
|
]
|
2026-02-28 00:18:43 +08:00
|
|
|
|
}
|
|
|
|
|
|
],
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"import numpy as np\n",
|
|
|
|
|
|
"import matplotlib.pyplot as plt\n",
|
|
|
|
|
|
"import random\n",
|
|
|
|
|
|
"from collections import defaultdict\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 1. 简单的 1D 走廊黑盒环境\n",
|
|
|
|
|
|
"def step(state, action):\n",
|
|
|
|
|
|
" if state == 3:\n",
|
|
|
|
|
|
" return 3, 0, True\n",
|
|
|
|
|
|
" if action == 1:\n",
|
|
|
|
|
|
" next_state = state + 1\n",
|
|
|
|
|
|
" else:\n",
|
|
|
|
|
|
" next_state = max(0, state - 1)\n",
|
|
|
|
|
|
" reward = 10 if next_state == 3 else -1\n",
|
|
|
|
|
|
" done = (next_state == 3)\n",
|
|
|
|
|
|
" return next_state, reward, done\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 2. epsilon-greedy 策略\n",
|
|
|
|
|
|
"def epsilon_greedy_policy(state, Q, epsilon, n_actions=2):\n",
|
|
|
|
|
|
" if random.uniform(0, 1) < epsilon:\n",
|
|
|
|
|
|
" return random.choice(range(n_actions))\n",
|
|
|
|
|
|
" else:\n",
|
|
|
|
|
|
" return np.argmax(Q[state])\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 3. 封装好的蒙特卡洛控制算法\n",
|
|
|
|
|
|
"def run_mc_control(num_episodes, gamma, initial_epsilon, decay_epsilon=False):\n",
|
|
|
|
|
|
" Q = defaultdict(lambda: np.zeros(2))\n",
|
|
|
|
|
|
" returns_sum = defaultdict(float)\n",
|
|
|
|
|
|
" returns_count = defaultdict(float)\n",
|
|
|
|
|
|
" episode_returns = [] # 记录每一局的最终回报，用来画图\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 循环执行多次回合\n",
|
|
|
|
|
|
" for i in range(num_episodes):\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # --- 【彩蛋核心逻辑：Epsilon 衰减】 ---\n",
|
|
|
|
|
|
" # 如果开启衰减，每一局的 epsilon 都会按照比例减小，但最低不低于 0.01\n",
|
|
|
|
|
|
" if decay_epsilon:\n",
|
|
|
|
|
|
" epsilon = max(0.01, initial_epsilon * (1 - i / num_episodes))\n",
|
|
|
|
|
|
" else:\n",
|
|
|
|
|
|
" epsilon = initial_epsilon\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" episode = []\n",
|
|
|
|
|
|
" state = 0\n",
|
|
|
|
|
|
" episode_return = 0\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # 跑完一个完整的回合\n",
|
|
|
|
|
|
" while True:\n",
|
|
|
|
|
|
" action = epsilon_greedy_policy(state, Q, epsilon)\n",
|
|
|
|
|
|
" next_state, reward, done = step(state, action)\n",
|
|
|
|
|
|
" episode.append((state, action, reward))\n",
|
|
|
|
|
|
" episode_return += reward\n",
|
|
|
|
|
|
" state = next_state\n",
|
|
|
|
|
|
" if done: break\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" episode_returns.append(episode_return)\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # 从最后一步倒算回报并更新 Q 表 (初次访问 MC)\n",
|
|
|
|
|
|
" G = 0.0\n",
|
|
|
|
|
|
" for t in reversed(range(len(episode))):\n",
|
|
|
|
|
|
" s, a, r = episode[t]\n",
|
|
|
|
|
|
" G = gamma * G + r\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" # 检查是否初次访问\n",
|
|
|
|
|
|
" is_first_visit = True\n",
|
|
|
|
|
|
" for prev_t in range(t):\n",
|
|
|
|
|
|
" if episode[prev_t][0] == s and episode[prev_t][1] == a:\n",
|
|
|
|
|
|
" is_first_visit = False\n",
|
|
|
|
|
|
" break\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" if is_first_visit:\n",
|
|
|
|
|
|
" returns_sum[(s, a)] += G\n",
|
|
|
|
|
|
" returns_count[(s, a)] += 1.0\n",
|
|
|
|
|
|
" Q[s][a] = returns_sum[(s, a)] / returns_count[(s, a)]\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
" return episode_returns\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# --- 4. 运行对比实验并绘图 ---\n",
|
|
|
|
|
|
"num_episodes = 500\n",
|
|
|
|
|
|
"gamma = 0.9\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 实验一：恒定 epsilon = 0.2\n",
|
|
|
|
|
|
"returns_const = run_mc_control(num_episodes, gamma, initial_epsilon=0.2, decay_epsilon=False)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 实验二：衰减 epsilon (初始值大胆设为 0.5，慢慢降到 0.01)\n",
|
|
|
|
|
|
"returns_decay = run_mc_control(num_episodes, gamma, initial_epsilon=0.5, decay_epsilon=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 计算滑动平均 (平滑曲线，看起来更直观)\n",
|
|
|
|
|
|
"def moving_average(a, n=20):\n",
|
|
|
|
|
|
" ret = np.cumsum(a, dtype=float)\n",
|
|
|
|
|
|
" ret[n:] = ret[n:] - ret[:-n]\n",
|
|
|
|
|
|
" return ret[n - 1:] / n\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 绘图\n",
|
|
|
|
|
|
"plt.figure(figsize=(10, 6))\n",
|
|
|
|
|
|
"plt.plot(moving_average(returns_const), label='Constant Epsilon (0.2)')\n",
|
|
|
|
|
|
"plt.plot(moving_average(returns_decay), label='Decaying Epsilon (0.5 -> 0.01)')\n",
|
|
|
|
|
|
"plt.title('Monte Carlo Control: Constant vs Decaying Epsilon')\n",
|
|
|
|
|
|
"plt.xlabel('Episodes (Smoothed over 20 episodes)')\n",
|
|
|
|
|
|
"plt.ylabel('Average Return per Episode')\n",
|
|
|
|
|
|
"plt.legend()\n",
|
|
|
|
|
|
"plt.grid(True)\n",
|
|
|
|
|
|
"plt.show()"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"kernelspec": {
|
|
|
|
|
|
"display_name": "GymRL",
|
|
|
|
|
|
"language": "python",
|
|
|
|
|
|
"name": "python3"
|
|
|
|
|
|
},
|
|
|
|
|
|
"language_info": {
|
|
|
|
|
|
"codemirror_mode": {
|
|
|
|
|
|
"name": "ipython",
|
|
|
|
|
|
"version": 3
|
|
|
|
|
|
},
|
|
|
|
|
|
"file_extension": ".py",
|
|
|
|
|
|
"mimetype": "text/x-python",
|
|
|
|
|
|
"name": "python",
|
|
|
|
|
|
"nbconvert_exporter": "python",
|
|
|
|
|
|
"pygments_lexer": "ipython3",
|
|
|
|
|
|
"version": "3.13.9"
|
|
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"nbformat": 4,
|
|
|
|
|
|
"nbformat_minor": 5
|
|
|
|
|
|
}
|
