每日算法 — 使用java实现Flappy Bird：Actor-Critic策略梯度与价值函数联合优化

引言

Flappy Bird是一款风靡全球的极简风格休闲游戏，玩家通过控制小鸟的跳跃来穿越管道间隙。本文将使用Java实现一个基于Actor-Critic强化学习的Flappy Bird AI，让AI自主学习何时跳跃、何时下落的决策策略。Actor-Critic算法结合了策略梯度（Policy Gradient）与价值函数逼近，是强化学习领域最经典的方法之一。

核心算法思路

Actor-Critic架构

Actor-Critic算法由两个神经网络组成：
– Actor（策略网络）：输入当前状态，输出动作概率分布（跳跃或不跳）
– Critic（价值网络）：输入当前状态，输出状态价值评估（预期累计奖励）

策略梯度更新

Actor网络通过策略梯度定理更新：
– 如果Critic评估某个动作能带来高于预期的好处（优势函数Advantage > 0），则增加该动作的概率
– 反之则降低该动作的概率

时序差分学习

Critic网络使用时序差分（TD）误差来更新价值估计，使预测值逐步逼近真实的累计回报。

Java实现

游戏环境模拟

import java.util.*;

/**
 * Flappy Bird游戏环境
 */
public class FlappyBirdEnv {
    private static final int SCREEN_WIDTH = 288;
    private static final int SCREEN_HEIGHT = 512;
    private static final int BIRD_X = 50;       // 小鸟固定x坐标
    private static final int BIRD_WIDTH = 34;
    private static final int BIRD_HEIGHT = 24;
    private static final int PIPE_WIDTH = 52;
    private static final int PIPE_GAP = 100;    // 管道间隙
    private static final int GRAVITY = 1;       // 重力加速度
    private static final int JUMP_VELOCITY = -9; // 跳跃初速度

    private double birdY;        // 小鸟y坐标
    private double birdVel;      // 小鸟垂直速度
    private List<Pipe> pipes;    // 管道列表
    private int score;
    private int frame;
    private boolean done;
    private Random random = new Random();

    public FlappyBirdEnv() {
        reset();
    }

    /**
     * 重置游戏
     */
    public void reset() {
        birdY = SCREEN_HEIGHT / 2.0;
        birdVel = 0;
        score = 0;
        frame = 0;
        done = false;
        pipes = new ArrayList<>();
        // 生成初始管道
        pipes.add(new Pipe(SCREEN_WIDTH + 100));
        pipes.add(new Pipe(SCREEN_WIDTH + 250));
    }

    /**
     * 执行动作
     * @param action 0=不操作（下落），1=跳跃
     * @return [下一状态, 即时奖励, 是否结束]
     */
    public Object[] step(int action) {
        if (done) {
            return new Object[]{getState(), 0.0, true};
        }

        // 执行动作
        if (action == 1) {
            birdVel = JUMP_VELOCITY;
        }

        // 物理更新
        birdVel += GRAVITY;
        birdY += birdVel;
        frame++;

        // 更新管道位置
        for (Pipe pipe : pipes) {
            pipe.x -= 2; // 管道向左移动
        }

        // 移除越界管道并生成新管道
        if (!pipes.isEmpty() && pipes.get(0).x < -PIPE_WIDTH) {
            pipes.remove(0);
            pipes.add(new Pipe(SCREEN_WIDTH));
            score++;
        }

        // 碰撞检测
        done = checkCollision();

        // 计算奖励
        double reward;
        if (done) {
            reward = -10.0; // 死亡惩罚
        } else {
            // 存活奖励 + 通过管道奖励
            reward = 0.1 + (frame % 90 == 0 ? 1.0 : 0);
        }

        return new Object[]{getState(), reward, done};
    }

    /**
     * 获取当前状态向量（用于神经网络输入）
     * 状态包含：小鸟y坐标归一化、小鸟速度归一化、
     * 下一管道水平距离、下一管道上沿高度、下一管道下沿高度
     */
    public double[] getState() {
        double[] state = new double[5];
        state[0] = birdY / SCREEN_HEIGHT;
        state[1] = (birdVel + 10) / 20.0; // 归一化到[0,1]

        // 找到下一根管道
        Pipe nextPipe = null;
        for (Pipe pipe : pipes) {
            if (pipe.x + PIPE_WIDTH >= BIRD_X) {
                nextPipe = pipe;
                break;
            }
        }

        if (nextPipe != null) {
            state[2] = (nextPipe.x - BIRD_X) / SCREEN_WIDTH;
            state[3] = nextPipe.gapY / SCREEN_HEIGHT;
            state[4] = (nextPipe.gapY + PIPE_GAP) / SCREEN_HEIGHT;
        } else {
            state[2] = 1.0;
            state[3] = 0.4;
            state[4] = 0.6;
        }

        return state;
    }

    private boolean checkCollision() {
        // 边界碰撞
        if (birdY < 0 || birdY + BIRD_HEIGHT > SCREEN_HEIGHT) {
            return true;
        }

        // 管道碰撞
        for (Pipe pipe : pipes) {
            if (BIRD_X + BIRD_WIDTH > pipe.x && BIRD_X < pipe.x + PIPE_WIDTH) {
                if (birdY < pipe.gapY || birdY + BIRD_HEIGHT > pipe.gapY + PIPE_GAP) {
                    return true;
                }
            }
        }
        return false;
    }

    public boolean isDone() { return done; }
    public int getScore() { return score; }
    public int getFrame() { return frame; }

    /**
     * 管道内部类
     */
    private class Pipe {
        double x;
        double gapY; // 间隙上沿y坐标

        Pipe(double x) {
            this.x = x;
            // 随机生成间隙位置（留出上下边距）
            this.gapY = 50 + random.nextInt(SCREEN_HEIGHT - PIPE_GAP - 150);
        }
    }
}

Actor-Critic神经网络

import java.util.Random;

/**
 * Actor-Critic神经网络
 */
public class ActorCriticNetwork {
    private Random random = new Random(42);
    private double learningRate = 0.001;
    private double gamma = 0.99; // 折扣因子

    // 共享隐藏层参数
    private double[][] w1; // 输入->隐藏层 [input_size][hidden_size]
    private double[] b1;   // 隐藏层偏置

    // Actor输出层参数
    private double[][] wActor; // 隐藏层->Actor [hidden_size][2]
    private double[] bActor;

    // Critic输出层参数
    private double[] wCritic; // 隐藏层->Critic [hidden_size]
    private double bCritic;

    private int inputSize, hiddenSize;

    // 前向传播缓存
    private double[] hiddenInput, hiddenOutput;
    private double[] actorInput, actorOutput;
    private double criticInput, criticOutput;

    public ActorCriticNetwork(int inputSize, int hiddenSize) {
        this.inputSize = inputSize;
        this.hiddenSize = hiddenSize;

        w1 = new double[inputSize][hiddenSize];
        b1 = new double[hiddenSize];
        wActor = new double[hiddenSize][2];
        bActor = new double[2];
        wCritic = new double[hiddenSize];
        bCritic = 0;

        initializeWeights();
    }

    private void initializeWeights() {
        double limit1 = Math.sqrt(2.0 / inputSize);
        for (int i = 0; i < inputSize; i++) {
            for (int j = 0; j < hiddenSize; j++) {
                w1[i][j] = random.nextGaussian() * limit1;
            }
        }

        double limitActor = Math.sqrt(2.0 / hiddenSize);
        for (int h = 0; h < hiddenSize; h++) {
            for (int a = 0; a < 2; a++) {
                wActor[h][a] = random.nextGaussian() * limitActor;
            }
            wCritic[h] = random.nextGaussian() * limitActor;
        }
    }

    /**
     * 前向传播
     * @return [actor_probs[2], critic_value]
     */
    public Object[] forward(double[] state) {
        // 共享隐藏层
        hiddenInput = new double[hiddenSize];
        hiddenOutput = new double[hiddenSize];
        for (int j = 0; j < hiddenSize; j++) {
            double sum = b1[j];
            for (int i = 0; i < inputSize; i++) {
                sum += state[i] * w1[i][j];
            }
            hiddenInput[j] = sum;
            hiddenOutput[j] = relu(sum);
        }

        // Actor输出
        actorInput = new double[2];
        actorOutput = new double[2];
        for (int a = 0; a < 2; a++) {
            double sum = bActor[a];
            for (int h = 0; h < hiddenSize; h++) {
                sum += hiddenOutput[h] * wActor[h][a];
            }
            actorInput[a] = sum;
        }
        actorOutput = softmax(actorInput);

        // Critic输出
        criticInput = bCritic;
        for (int h = 0; h < hiddenSize; h++) {
            criticInput += hiddenOutput[h] * wCritic[h];
        }
        criticOutput = criticInput; // 线性输出

        return new Object[]{actorOutput, criticOutput};
    }

    /**
     * 根据Actor策略选择动作
     */
    public int selectAction(double[] state) {
        Object[] result = forward(state);
        double[] probs = (double[]) result[0];
        // 按概率采样
        double r = random.nextDouble();
        return r < probs[0] ? 0 : 1;
    }

    /**
     * 训练一步
     * @param state 当前状态
     * @param action 执行的动作
     * @param reward 即时奖励
     * @param nextState 下一状态
     * @param done 是否结束
     */
    public void train(double[] state, int action, double reward, 
                       double[] nextState, boolean done) {
        // 前向传播当前状态
        Object[] current = forward(state);
        double[] actorProbs = (double[]) current[0];
        double currentValue = (Double) current[1];

        // 前向传播下一状态
        Object[] next = forward(nextState);
        double nextValue = done ? 0 : (Double) next[1];

        // TD误差（优势函数近似）
        double tdTarget = reward + gamma * nextValue;
        double tdError = tdTarget - currentValue;

        // === Critic更新 ===
        // 价值损失梯度 dL/dv = -(tdTarget - v)
        double criticGrad = -tdError;

        // === Actor更新（策略梯度）===
        // 策略梯度: dlog(pi(a)) / dtheta * Advantage
        double[] actorGrad = new double[2];
        for (int a = 0; a < 2; a++) {
            if (a == action) {
                actorGrad[a] = (1 - actorProbs[a]) * tdError; // 执行的动作
            } else {
                actorGrad[a] = -actorProbs[a] * tdError; // 未执行的动作
            }
        }

        // === 反向传播 ===
        // 1. 计算隐藏层梯度
        double[] hiddenGrad = new double[hiddenSize];
        for (int h = 0; h < hiddenSize; h++) {
            double grad = 0;
            // Actor贡献
            for (int a = 0; a < 2; a++) {
                grad += actorGrad[a] * wActor[h][a];
            }
            // Critic贡献
            grad += criticGrad * wCritic[h];
            // ReLU导数
            hiddenGrad[h] = grad * reluDerivative(hiddenInput[h]);
        }

        // 2. 更新Actor权重
        for (int h = 0; h < hiddenSize; h++) {
            for (int a = 0; a < 2; a++) {
                wActor[h][a] -= learningRate * actorGrad[a] * hiddenOutput[h];
            }
        }
        for (int a = 0; a < 2; a++) {
            bActor[a] -= learningRate * actorGrad[a];
        }

        // 3. 更新Critic权重
        for (int h = 0; h < hiddenSize; h++) {
            wCritic[h] -= learningRate * criticGrad * hiddenOutput[h];
        }
        bCritic -= learningRate * criticGrad;

        // 4. 更新共享隐藏层权重
        for (int i = 0; i < inputSize; i++) {
            for (int h = 0; h < hiddenSize; h++) {
                w1[i][h] -= learningRate * hiddenGrad[h] * state[i];
            }
        }
        for (int h = 0; h < hiddenSize; h++) {
            b1[h] -= learningRate * hiddenGrad[h];
        }
    }

    private double relu(double x) {
        return Math.max(0, x);
    }

    private double reluDerivative(double x) {
        return x > 0 ? 1.0 : 0.0;
    }

    private double[] softmax(double[] x) {
        double max = Math.max(x[0], x[1]);
        double exp0 = Math.exp(x[0] - max);
        double exp1 = Math.exp(x[1] - max);
        double sum = exp0 + exp1;
        return new double[]{exp0 / sum, exp1 / sum};
    }

    public void setLearningRate(double lr) { this.learningRate = lr; }
}

训练引擎

/**
 * Actor-Critic训练引擎
 */
public class ActorCriticTrainer {
    private ActorCriticNetwork network;
    private FlappyBirdEnv env;

    public ActorCriticTrainer(ActorCriticNetwork network) {
        this.network = network;
        this.env = new FlappyBirdEnv();
    }

    /**
     * 训练一个episode
     */
    public void trainEpisode() {
        env.reset();
        double[] state = env.getState();
        int steps = 0;
        final int MAX_STEPS = 10000;

        while (!env.isDone() && steps < MAX_STEPS) {
            int action = network.selectAction(state);
            Object[] result = env.step(action);
            double[] nextState = (double[]) result[0];
            double reward = (Double) result[1];
            boolean done = (Boolean) result[2];

            network.train(state, action, reward, nextState, done);

            state = nextState;
            steps++;
        }
    }

    /**
     * 批量训练
     */
    public void train(int episodes) {
        for (int ep = 0; ep < episodes; ep++) {
            trainEpisode();

            if ((ep + 1) % 100 == 0) {
                double avgScore = evaluate(10);
                System.out.printf("Episode %d: 平均得分 %.1f%n", ep + 1, avgScore);
            }
        }
    }

    /**
     * 评估当前策略
     */
    public double evaluate(int games) {
        int totalScore = 0;
        for (int g = 0; g < games; g++) {
            env.reset();
            double[] state = env.getState();
            while (!env.isDone()) {
                Object[] result = network.forward(state);
                double[] probs = (double[]) result[0];
                int action = probs[1] > probs[0] ? 1 : 0; // 贪婪选择
                Object[] stepResult = env.step(action);
                state = (double[]) stepResult[0];
            }
            totalScore += env.getScore();
        }
        return totalScore / (double) games;
    }
}

主程序入口

/**
 * Flappy Bird Actor-Critic AI主程序
 */
public class FlappyBirdAI {
    public static void main(String[] args) {
        // 创建Actor-Critic网络
        // 输入: 5维状态 [birdY_norm, birdVel_norm, pipeDist, pipeTop, pipeBottom]
        // 隐藏层: 64个神经元
        ActorCriticNetwork network = new ActorCriticNetwork(5, 64);
        network.setLearningRate(0.001);

        System.out.println("开始Actor-Critic训练...");
        ActorCriticTrainer trainer = new ActorCriticTrainer(network);
        trainer.train(5000);

        System.out.println("\n训练完成！进行最终评估...");
        double finalScore = trainer.evaluate(50);
        System.out.printf("最终平均得分: %.1f%n", finalScore);

        // 演示一局
        System.out.println("\n=== AI演示 ===");
        FlappyBirdEnv demoEnv = new FlappyBirdEnv();
        demoEnv.reset();
        double[] state = demoEnv.getState();
        int frame = 0;

        while (!demoEnv.isDone() && frame < 5000) {
            Object[] result = network.forward(state);
            double[] probs = (double[]) result[0];
            double value = (Double) result[1];
            int action = probs[1] > probs[0] ? 1 : 0;

            if (frame % 30 == 0) {
                System.out.printf("Frame %d: 跳跃概率=%.3f, 价值=%.3f, 动作=%s%n",
                    frame, probs[1], value, action == 1 ? "跳跃" : "下落");
            }

            Object[] stepResult = demoEnv.step(action);
            state = (double[]) stepResult[0];
            frame++;
        }

        System.out.println("\n演示结束！得分: " + demoEnv.getScore());
    }
}

算法复杂度分析

其中：I=5（输入维度），H=64（隐藏层），A=2（动作数），S=每局最大步数，N=训练轮数。

总结

本文通过Java实现了基于Actor-Critic的Flappy Bird AI，核心要点包括：

• Actor网络学习策略，输出动作概率分布
• Critic网络评估价值，提供TD误差指导策略更新
• 共享隐藏层让状态表示同时服务于策略和价值估计
• 时序差分学习使价值估计逐步逼近真实回报

读者可以尝试调整网络结构（增加隐藏层维度、使用RNN处理时序信息）、改进奖励函数（引入管道中心对齐奖励）或探索更先进的PPO、A3C等策略梯度变体算法。