围棋是起源于中国的古老棋类游戏，以其极其庞大的状态空间著称。传统基于搜索的AI难以应对围棋的复杂度，而深度强化学习为围棋AI开辟了全新道路。本文用Java实现一个基于PPO（Proximal Policy Optimization）算法的围棋AI，通过自我对弈不断迭代提升策略水平。
一、PPO算法核心思想
PPO是一种策略梯度方法，通过限制策略更新幅度来保证训练稳定性。其核心是裁剪目标函数：
L^{CLIP}(θ) = E_t [min(r_t(θ) Â_t, clip(r_t(θ), 1-ε, 1+ε) Â_t)]
其中 r_t(θ) 是新旧策略的概率比，Â_t 是优势函数估计。裁剪操作防止策略在一次更新中发生剧烈变化。
二、围棋状态表示
围棋棋盘为19×19，每个交叉点有三种状态：黑子、白子、空。使用三维数组表示棋盘状态，同时记录气、提子等信息用于规则判定。
三、Java核心实现
import java.util.*;
/**
* 围棋PPO自我对弈AI核心实现
*/
public class GoPPOAI {
// 棋盘大小
public static final int BOARD_SIZE = 19;
public static final int EMPTY = 0;
public static final int BLACK = 1;
public static final int WHITE = 2;
// PPO超参数
private static final double CLIP_EPSILON = 0.2;
private static final double LEARNING_RATE = 0.0003;
private static final double GAMMA = 0.99; // 折扣因子
private static final double LAMBDA = 0.95; // GAE参数
private static final int EPOCHS = 4;
private static final int BATCH_SIZE = 64;
// 策略网络和价值网络（简化版使用全连接网络）
private NeuralNetwork policyNet;
private NeuralNetwork valueNet;
public GoPPOAI() {
// 输入：19*19*3 (当前棋子、对手棋子、上一步落子位置)
this.policyNet = new NeuralNetwork(BOARD_SIZE * BOARD_SIZE * 3,
new int[]{256, 256},
BOARD_SIZE * BOARD_SIZE + 1); // +1 for pass
this.valueNet = new NeuralNetwork(BOARD_SIZE * BOARD_SIZE * 3,
new int[]{256, 128},
1);
}
/**
* 棋盘状态类
*/
static class BoardState {
int[][] board = new int[BOARD_SIZE][BOARD_SIZE];
int currentPlayer = BLACK;
int lastMoveX = -1, lastMoveY = -1;
/**
* 获取当前玩家的合法落子位置
*/
List<int[]> getLegalMoves() {
List<int[]> moves = new ArrayList<>();
for (int i = 0; i < BOARD_SIZE; i++) {
for (int j = 0; j < BOARD_SIZE; j++) {
if (board[i][j] == EMPTY && !isSuicide(i, j)) {
moves.add(new int[]{i, j});
}
}
}
// 允许pass
moves.add(new int[]{-1, -1});
return moves;
}
/**
* 判断是否为自杀手（简化版，未考虑打劫）
*/
boolean isSuicide(int x, int y) {
if (board[x][y] != EMPTY) return false;
// 简化判断：如果落子后该子没有气，则为自杀
// 实际实现需要更复杂的连通块气计算
int[][] temp = copyBoard();
temp[x][y] = currentPlayer;
return getLiberties(temp, x, y) == 0;
}
/**
* 计算某位置连通块的气数（简化版）
*/
int getLiberties(int[][] b, int x, int y) {
int color = b[x][y];
if (color == EMPTY) return 0;
boolean[][] visited = new boolean[BOARD_SIZE][BOARD_SIZE];
Queue<int[]> q = new LinkedList<>();
q.add(new int[]{x, y});
visited[x][y] = true;
int liberties = 0;
int[][] dirs = {{-1,0},{1,0},{0,-1},{0,1}};
while (!q.isEmpty()) {
int[] cur = q.poll();
for (int[] d : dirs) {
int nx = cur[0] + d[0], ny = cur[1] + d[1];
if (nx < 0 || nx >= BOARD_SIZE || ny < 0 || ny >= BOARD_SIZE) continue;
if (b[nx][ny] == EMPTY) {
liberties++;
} else if (b[nx][ny] == color && !visited[nx][ny]) {
visited[nx][ny] = true;
q.add(new int[]{nx, ny});
}
}
}
return liberties;
}
int[][] copyBoard() {
int[][] copy = new int[BOARD_SIZE][BOARD_SIZE];
for (int i = 0; i < BOARD_SIZE; i++) {
System.arraycopy(board[i], 0, copy[i], 0, BOARD_SIZE);
}
return copy;
}
/**
* 执行落子并返回新状态
*/
BoardState makeMove(int x, int y) {
BoardState next = new BoardState();
for (int i = 0; i < BOARD_SIZE; i++) {
System.arraycopy(this.board[i], 0, next.board[i], 0, BOARD_SIZE);
}
next.currentPlayer = 3 - this.currentPlayer; // 切换玩家
if (x >= 0 && y >= 0) {
next.board[x][y] = this.currentPlayer;
next.lastMoveX = x;
next.lastMoveY = y;
}
return next;
}
/**
* 将棋盘转为神经网络的输入向量
*/
double[] toInput() {
double[] input = new double[BOARD_SIZE * BOARD_SIZE * 3];
int idx = 0;
for (int i = 0; i < BOARD_SIZE; i++) {
for (int j = 0; j < BOARD_SIZE; j++) {
input[idx++] = (board[i][j] == currentPlayer) ? 1.0 : 0.0;
input[idx++] = (board[i][j] == 3 - currentPlayer) ? 1.0 : 0.0;
input[idx++] = (i == lastMoveX && j == lastMoveY) ? 1.0 : 0.0;
}
}
return input;
}
}
/**
* 简化的神经网络实现（全连接层）
*/
static class NeuralNetwork {
private final List<double[][]> weights;
private final List<double[]> biases;
private final Random rand = new Random(42);
NeuralNetwork(int inputSize, int[] hiddenSizes, int outputSize) {
weights = new ArrayList<>();
biases = new ArrayList<>();
int prevSize = inputSize;
for (int hiddenSize : hiddenSizes) {
weights.add(initMatrix(prevSize, hiddenSize));
biases.add(new double[hiddenSize]);
prevSize = hiddenSize;
}
weights.add(initMatrix(prevSize, outputSize));
biases.add(new double[outputSize]);
}
double[][] initMatrix(int rows, int cols) {
double[][] m = new double[rows][cols];
double scale = Math.sqrt(2.0 / rows);
for (int i = 0; i < rows; i++) {
for (int j = 0; j < cols; j++) {
m[i][j] = rand.nextGaussian() * scale;
}
}
return m;
}
/**
* 前向传播
*/
double[] forward(double[] input) {
double[] current = input;
for (int i = 0; i < weights.size(); i++) {
double[] next = new double[biases.get(i).length];
for (int j = 0; j < next.length; j++) {
double sum = biases.get(i)[j];
for (int k = 0; k < current.length; k++) {
sum += current[k] * weights.get(i)[k][j];
}
// ReLU激活，最后一层除外
if (i < weights.size() - 1) {
next[j] = Math.max(0, sum);
} else {
next[j] = sum;
}
}
current = next;
}
return current;
}
/**
* 策略网络输出softmax概率
*/
double[] policyForward(double[] input) {
double[] logits = forward(input);
double[] probs = new double[logits.length];
double max = Double.NEGATIVE_INFINITY;
for (double v : logits) max = Math.max(max, v);
double sum = 0;
for (int i = 0; i < logits.length; i++) {
probs[i] = Math.exp(logits[i] - max);
sum += probs[i];
}
for (int i = 0; i < probs.length; i++) {
probs[i] /= sum;
}
return probs;
}
}
/**
* 经验回放缓冲区
*/
static class ExperienceBuffer {
List<double[]> states = new ArrayList<>();
List<Integer> actions = new ArrayList<>();
List<Double> rewards = new ArrayList<>();
List<Double> values = new ArrayList<>();
List<Double> logProbs = new ArrayList<>();
List<Boolean> dones = new ArrayList<>();
void add(double[] state, int action, double reward, double value, double logProb, boolean done) {
states.add(state.clone());
actions.add(action);
rewards.add(reward);
values.add(value);
logProbs.add(logProb);
dones.add(done);
}
int size() { return states.size(); }
void clear() {
states.clear(); actions.clear(); rewards.clear();
values.clear(); logProbs.clear(); dones.clear();
}
}
/**
* 使用GAE计算优势函数
*/
double[] computeAdvantages(ExperienceBuffer buffer) {
int n = buffer.size();
double[] advantages = new double[n];
double[] returns = new double[n];
double gae = 0;
for (int i = n - 1; i >= 0; i--) {
double nextValue = (i == n - 1) ? 0 : buffer.values.get(i + 1);
double delta = buffer.rewards.get(i) + GAMMA * nextValue * (buffer.dones.get(i) ? 0 : 1) - buffer.values.get(i);
gae = delta + GAMMA * LAMBDA * (buffer.dones.get(i) ? 0 : 1) * gae;
advantages[i] = gae;
returns[i] = advantages[i] + buffer.values.get(i);
}
return advantages;
}
/**
* 自我对弈一局，收集经验数据
*/
ExperienceBuffer selfPlay() {
ExperienceBuffer buffer = new ExperienceBuffer();
BoardState state = new BoardState();
int moveCount = 0;
int passCount = 0;
while (moveCount < 361 * 2 && passCount < 2) {
List<int[]> legalMoves = state.getLegalMoves();
if (legalMoves.isEmpty()) break;
double[] input = state.toInput();
double[] probs = policyNet.policyForward(input);
double value = valueNet.forward(input)[0];
// 屏蔽非法动作
double[] mask = new double[probs.length];
Arrays.fill(mask, -1e9);
for (int[] m : legalMoves) {
int idx = (m[0] == -1) ? probs.length - 1 : m[0] * BOARD_SIZE + m[1];
mask[idx] = probs[idx];
}
// 从合法动作中采样
double sum = 0;
for (double v : mask) if (v > -1e8) sum += Math.exp(v);
double rand = Math.random() * sum;
int action = 0;
double cumsum = 0;
for (int i = 0; i < mask.length; i++) {
if (mask[i] > -1e8) {
cumsum += Math.exp(mask[i]);
if (cumsum >= rand) {
action = i;
break;
}
}
}
double logProb = Math.log(probs[action] + 1e-10);
int x = (action == probs.length - 1) ? -1 : action / BOARD_SIZE;
int y = (action == probs.length - 1) ? -1 : action % BOARD_SIZE;
BoardState nextState = state.makeMove(x, y);
boolean done = false;
// 简单终局判断：连续两次pass
if (x == -1) {
passCount++;
if (passCount >= 2) done = true;
} else {
passCount = 0;
}
// 简化奖励：终局时根据领地计算
double reward = 0;
if (done) {
reward = evaluateTerritory(nextState);
}
buffer.add(input, action, reward, value, logProb, done);
state = nextState;
moveCount++;
}
return buffer;
}
/**
* 简化版领地评估（数子法）
*/
double evaluateTerritory(BoardState state) {
int blackCount = 0, whiteCount = 0;
for (int i = 0; i < BOARD_SIZE; i++) {
for (int j = 0; j < BOARD_SIZE; j++) {
if (state.board[i][j] == BLACK) blackCount++;
else if (state.board[i][j] == WHITE) whiteCount++;
}
}
// 贴目简化处理（黑贴3.75目）
double result = (blackCount - whiteCount - 3.75);
// 从当前玩家视角返回奖励
return (state.currentPlayer == BLACK) ? result / 100.0 : -result / 100.0;
}
/**
* 主训练循环
*/
void train(int iterations) {
for (int iter = 0; iter < iterations; iter++) {
ExperienceBuffer buffer = selfPlay();
if (buffer.size() == 0) continue;
double[] advantages = computeAdvantages(buffer);
// 归一化优势
double mean = 0, std = 0;
for (double a : advantages) mean += a;
mean /= advantages.length;
for (double a : advantages) std += (a - mean) * (a - mean);
std = Math.sqrt(std / advantages.length) + 1e-8;
for (int i = 0; i < advantages.length; i++) {
advantages[i] = (advantages[i] - mean) / std;
}
System.out.println("Iteration " + iter + ", Game length: " + buffer.size());
// PPO更新（简化版，实际应使用反向传播）
// 这里展示核心逻辑框架
for (int epoch = 0; epoch < EPOCHS; epoch++) {
// 对收集的经验进行多轮小批量更新
// 计算策略损失和价值损失
// L_clip = -E[min(r*A, clip(r)*A)]
// L_value = E[(V_target - V_pred)^2]
// 实际实现需要使用自动微分框架
}
buffer.clear();
}
}
public static void main(String[] args) {
GoPPOAI ai = new GoPPOAI();
ai.train(1000);
}
}
四、关键设计要点
策略网络输出每个合法落子位置的概率分布，指导AI选择落子点。网络结构接收棋盘状态的三通道表示，经过多层全连接后输出361+1维的策略向量。
价值网络评估当前局面的胜率期望，为PPO提供状态价值估计。价值估计与GAE（Generalized Advantage Estimation）结合，降低策略梯度估计的方差。
自我对弈是训练数据的主要来源。AI左右互搏，每局结束后根据终局结果分配奖励，通过PPO的裁剪目标函数稳定地更新策略。
五、复杂度分析
策略网络前向传播的时间复杂度为 O(H * W * D + hidden_layers)，其中 H=W=19 为棋盘尺寸，D=3 为输入通道数。自我对弈一局约产生200-400条经验，PPO每次迭代的更新复杂度与经验数量和网络参数量成正比。
六、扩展方向
实际应用中，策略网络通常采用ResNet风格的卷积神经网络以更好地利用围棋的空间结构。此外，可以引入MCTS与神经网络结合（如AlphaZero），让搜索过程指导策略改进，实现更强的棋力。分布式训练框架和更大的网络规模也是提升AI水平的关键。
