引言
Flappy Bird是一款看似简单却极具挑战的休闲游戏。传统规则下玩家需要精准控制小鸟的跳跃时机穿越管道。本文用Java实现DQN（Deep Q-Network）深度强化学习算法，让AI通过经验回放自动学习最优跳跃策略，无需任何人工特征工程。
DQN核心原理
DQN将Q-Learning与深度神经网络结合，用神经网络近似Q值函数。关键创新包括：
– 经验回放（Experience Replay）：存储历史转移样本，随机抽样打破数据相关性
– 目标网络（Target Network）：独立的目标Q网络稳定训练过程
– ε-贪心探索：平衡探索与利用
Java实现
import java.util.*;
/**
* Flappy Bird DQN智能体
* 使用深度Q网络学习最优跳跃策略
*/
public class FlappyBirdDQN {
// 神经网络参数
private final int stateSize = 4; // 输入特征维度
private final int hiddenSize = 24; // 隐藏层神经元数
private final int actionSize = 2; // 动作空间：0=不跳，1=跳跃
// 主网络与目标网络权重
private final double[][] W1, W1Target; // 输入->隐藏 (stateSize x hiddenSize)
private final double[] b1, b1Target; // 隐藏层偏置
private final double[][] W2, W2Target; // 隐藏->输出 (hiddenSize x actionSize)
private final double[] b2, b2Target; // 输出层偏置
// 超参数
private final double gamma = 0.95; // 折扣因子
private final double epsilon = 1.0; // 初始探索率
private final double epsilonMin = 0.01;
private final double epsilonDecay = 0.995;
private final double learningRate = 0.001;
// 经验回放缓冲区
private final List<Experience> memory;
private final int memoryCapacity = 2000;
private final int batchSize = 32;
private double currentEpsilon;
private int trainingStep = 0;
private final int targetUpdateFreq = 100; // 每100步同步目标网络
public FlappyBirdDQN() {
// 初始化主网络权重（Xavier初始化）
W1 = xavierInit(stateSize, hiddenSize);
b1 = new double[hiddenSize];
W2 = xavierInit(hiddenSize, actionSize);
b2 = new double[actionSize];
// 目标网络初始与主网络相同
W1Target = deepCopy(W1);
b1Target = b1.clone();
W2Target = deepCopy(W2);
b2Target = b2.clone();
memory = new ArrayList<>();
currentEpsilon = epsilon;
}
/**
* 根据当前状态选择动作
* @param state [鸟高度, 鸟速度, 下管道距离, 上下管道开口中心]
* @return 0=不跳，1=跳跃
*/
public int act(double[] state) {
// ε-贪心策略
if (Math.random() <= currentEpsilon) {
return (int) (Math.random() * actionSize); // 随机探索
}
// 利用：选择Q值最大的动作
double[] qValues = forward(state, W1, b1, W2, b2);
return qValues[0] > qValues[1] ? 0 : 1;
}
/**
* 存储经验到回放缓冲区
*/
public void remember(double[] state, int action, double reward,
double[] nextState, boolean done) {
if (memory.size() >= memoryCapacity) {
memory.remove(0); // 淘汰最旧的经验
}
memory.add(new Experience(state, action, reward, nextState, done));
}
/**
* 从经验回放中随机抽样训练网络
*/
public void replay() {
if (memory.size() < batchSize) return;
// 随机抽样小批量经验
List<Experience> batch = sampleBatch();
for (Experience exp : batch) {
// 当前状态Q值
double[] qCurrent = forward(exp.state, W1, b1, W2, b2);
// 目标Q值：reward + gamma * max(Q_target(next_state))
double target = exp.reward;
if (!exp.done) {
double[] qNext = forward(exp.nextState, W1Target, b1Target,
W2Target, b2Target);
target += gamma * max(qNext);
}
// 仅更新所选动作的Q值，其余不变
double[] targetQ = qCurrent.clone();
targetQ[exp.action] = target;
// 反向传播更新主网络
backward(exp.state, qCurrent, targetQ);
}
// 衰减探索率
if (currentEpsilon > epsilonMin) {
currentEpsilon *= epsilonDecay;
}
// 定期同步目标网络
trainingStep++;
if (trainingStep % targetUpdateFreq == 0) {
syncTargetNetwork();
}
}
/**
* 前向传播：ReLU隐藏层 + 线性输出层
*/
private double[] forward(double[] state, double[][] w1, double[] b1,
double[][] w2, double[] b2) {
// 隐藏层：state @ W1 + b1，ReLU激活
double[] hidden = new double[hiddenSize];
for (int i = 0; i < hiddenSize; i++) {
double sum = b1[i];
for (int j = 0; j < stateSize; j++) {
sum += state[j] * w1[j][i];
}
hidden[i] = Math.max(0, sum); // ReLU
}
// 输出层：hidden @ W2 + b2（线性）
double[] output = new double[actionSize];
for (int i = 0; i < actionSize; i++) {
double sum = b2[i];
for (int j = 0; j < hiddenSize; j++) {
sum += hidden[j] * w2[j][i];
}
output[i] = sum;
}
return output;
}
/**
* 反向传播（简化版梯度下降）
*/
private void backward(double[] state, double[] pred, double[] target) {
// 输出层误差
double[] dOutput = new double[actionSize];
for (int i = 0; i < actionSize; i++) {
dOutput[i] = 2 * (pred[i] - target[i]) / actionSize;
}
// 隐藏层前向值（需重新计算）
double[] hidden = new double[hiddenSize];
for (int i = 0; i < hiddenSize; i++) {
double sum = b1[i];
for (int j = 0; j < stateSize; j++) {
sum += state[j] * W1[j][i];
}
hidden[i] = Math.max(0, sum);
}
// 更新W2, b2
for (int i = 0; i < actionSize; i++) {
for (int j = 0; j < hiddenSize; j++) {
W2[j][i] -= learningRate * dOutput[i] * hidden[j];
}
b2[i] -= learningRate * dOutput[i];
}
// 隐藏层误差（ReLU导数）
double[] dHidden = new double[hiddenSize];
for (int i = 0; i < hiddenSize; i++) {
double sum = 0;
for (int j = 0; j < actionSize; j++) {
sum += dOutput[j] * W2[i][j];
}
dHidden[i] = hidden[i] > 0 ? sum : 0; // ReLU导数
}
// 更新W1, b1
for (int i = 0; i < hiddenSize; i++) {
for (int j = 0; j < stateSize; j++) {
W1[j][i] -= learningRate * dHidden[i] * state[j];
}
b1[i] -= learningRate * dHidden[i];
}
}
private void syncTargetNetwork() {
for (int i = 0; i < W1.length; i++) {
W1Target[i] = W1[i].clone();
}
System.arraycopy(b1, 0, b1Target, 0, b1.length);
for (int i = 0; i < W2.length; i++) {
W2Target[i] = W2[i].clone();
}
System.arraycopy(b2, 0, b2Target, 0, b2.length);
}
private List<Experience> sampleBatch() {
List<Experience> batch = new ArrayList<>();
Random rand = new Random();
for (int i = 0; i < batchSize; i++) {
batch.add(memory.get(rand.nextInt(memory.size())));
}
return batch;
}
private double max(double[] arr) {
double m = arr[0];
for (double v : arr) if (v > m) m = v;
return m;
}
private double[][] xavierInit(int inSize, int outSize) {
double[][] w = new double[inSize][outSize];
double scale = Math.sqrt(2.0 / (inSize + outSize));
Random rand = new Random(42);
for (int i = 0; i < inSize; i++) {
for (int j = 0; j < outSize; j++) {
w[i][j] = rand.nextGaussian() * scale;
}
}
return w;
}
private double[][] deepCopy(double[][] src) {
double[][] dst = new double[src.length][];
for (int i = 0; i < src.length; i++) {
dst[i] = src[i].clone();
}
return dst;
}
private static class Experience {
final double[] state, nextState;
final int action;
final double reward;
final boolean done;
Experience(double[] s, int a, double r, double[] ns, boolean d) {
state = s; action = a; reward = r; nextState = ns; done = d;
}
}
/**
* 环境模拟：简化的Flappy Bird状态转移
*/
public static class FlappyEnv {
double birdY = 0.5; // 鸟的高度（0~1归一化）
double birdVel = 0; // 鸟的速度
double pipeX = 1.0; // 管道的水平位置
double pipeGapY = 0.5; // 管道开口中心
/**
* 执行动作，返回 [nextState, reward, done]
*/
public Object[] step(int action) {
// 动作1=跳跃，给予向上速度
if (action == 1) birdVel = -0.05;
// 物理更新
birdVel += 0.003; // 重力
birdY += birdVel;
pipeX -= 0.01; // 管道左移
boolean done = birdY < 0 || birdY > 1;
double reward = 0.01; // 存活奖励
// 碰撞检测（简化）
if (pipeX > 0.05 && pipeX < 0.15) {
if (birdY < pipeGapY - 0.08 || birdY > pipeGapY + 0.08) {
done = true;
reward = -1.0; // 碰撞惩罚
}
}
// 通过管道奖励
if (pipeX < -0.1) {
reward = 1.0;
pipeX = 1.0;
pipeGapY = 0.2 + Math.random() * 0.6; // 新管道随机开口
}
double[] state = {birdY, birdVel, pipeX, pipeGapY};
return new Object[]{state, reward, done};
}
public double[] reset() {
birdY = 0.5; birdVel = 0; pipeX = 1.0; pipeGapY = 0.5;
return new double[]{birdY, birdVel, pipeX, pipeGapY};
}
}
public static void main(String[] args) {
FlappyBirdDQN agent = new FlappyBirdDQN();
FlappyEnv env = new FlappyEnv();
int episodes = 500;
for (int ep = 0; ep < episodes; ep++) {
double[] state = env.reset();
int score = 0;
while (true) {
int action = agent.act(state);
Object[] result = env.step(action);
double[] nextState = (double[]) result[0];
double reward = (double) result[1];
boolean done = (boolean) result[2];
agent.remember(state, action, reward, nextState, done);
agent.replay();
state = nextState;
if (reward > 0.5) score++;
if (done) break;
}
if (ep % 50 == 0) {
System.out.println("Episode " + ep + " Score: " + score
+ " Epsilon: " + String.format("%.3f", agent.currentEpsilon));
}
}
}
}
训练技巧
- 奖励塑形：存活给予小正奖励，通过管道大奖励，碰撞大惩罚，引导智能体主动穿越管道
- 帧跳过（Frame Skip）：每4帧执行一次动作决策，加速训练且保持控制精度
- 状态归一化：所有输入特征缩放至[0,1]区间，提升神经网络收敛速度
总结
本文用纯Java实现了DQN的核心组件：ε-贪心策略、经验回放缓冲区、目标网络同步。通过将Flappy Bird的状态抽象为4维向量，AI无需人工特征设计即可自主学习跳跃时机。该框架可扩展至更复杂的Atari游戏。
