五子棋规则简单却蕴含丰富的策略深度，是研究博弈AI的经典平台。不同于传统的基于规则评估或蒙特卡洛树搜索的方法，本文采用Actor-Critic架构结合策略梯度方法，用Java实现一个能够通过自我对弈持续学习的五子棋AI系统。
一、Actor-Critic架构原理
Actor-Critic结合了策略梯度（Policy Gradient）和价值函数估计的优点：
- Actor（策略网络）：输出每个落子位置的概率分布，决定AI的行为策略
- Critic（价值网络）：评估当前局面的胜率期望，为Actor提供优势估计
两者共享部分网络层以加速训练，又分别输出策略头和价值头。训练时，Critic提供比蒙特卡洛回报更低方差的优势估计，使Actor的策略更新更加稳定。
二、五子棋特征工程
将15×15的棋盘转化为神经网络的输入需要精心设计的特征表示。本文采用多通道平面表示：当前玩家棋子平面、对手棋子平面、合法落子位置平面、以及历史落子热度平面。
三、Java核心实现
import java.util.*;
/**
* 五子棋Actor-Critic自我对弈AI
*/
public class GomokuActorCritic {
static final int BOARD_SIZE = 15;
static final int EMPTY = 0;
static final int BLACK = 1;
static final int WHITE = 2;
// 训练超参数
static final double LEARNING_RATE = 0.001;
static final double GAMMA = 0.99;
static final double ENTROPY_COEF = 0.01; // 熵正则化系数
static final int SELF_PLAY_GAMES = 100;
static final int TRAIN_EPOCHS = 10;
// 神经网络
private ActorCriticNetwork network;
private Optimizer optimizer;
public GomokuActorCritic() {
// 输入: 4通道 x 15 x 15
this.network = new ActorCriticNetwork(4, BOARD_SIZE, 128);
this.optimizer = new Optimizer(LEARNING_RATE);
}
/**
* 棋盘状态类
*/
static class Board {
int[][] grid = new int[BOARD_SIZE][BOARD_SIZE];
int currentPlayer = BLACK;
List<int[]> moveHistory = new ArrayList<>();
Board copy() {
Board b = new Board();
for (int i = 0; i < BOARD_SIZE; i++) {
System.arraycopy(this.grid[i], 0, b.grid[i], 0, BOARD_SIZE);
}
b.currentPlayer = this.currentPlayer;
b.moveHistory.addAll(this.moveHistory);
return b;
}
/**
* 获取所有合法落子位置
*/
List<int[]> getLegalMoves() {
List<int[]> moves = new ArrayList<>();
// 优先搜索已有棋子周围的位置（启发式）
boolean[][] near = new boolean[BOARD_SIZE][BOARD_SIZE];
for (int[] m : moveHistory) {
for (int dr = -2; dr <= 2; dr++) {
for (int dc = -2; dc <= 2; dc++) {
int nr = m[0] + dr, nc = m[1] + dc;
if (nr >= 0 && nr < BOARD_SIZE && nc >= 0 && nc < BOARD_SIZE) {
near[nr][nc] = true;
}
}
}
}
for (int r = 0; r < BOARD_SIZE; r++) {
for (int c = 0; c < BOARD_SIZE; c++) {
if (grid[r][c] == EMPTY && (moveHistory.isEmpty() || near[r][c])) {
moves.add(new int[]{r, c});
}
}
}
return moves;
}
/**
* 执行落子
*/
void makeMove(int r, int c) {
grid[r][c] = currentPlayer;
moveHistory.add(new int[]{r, c});
currentPlayer = 3 - currentPlayer;
}
/**
* 检查是否五连
*/
boolean checkWin(int r, int c, int player) {
int[][] dirs = {{0,1},{1,0},{1,1},{1,-1}};
for (int[] d : dirs) {
int count = 1;
// 正向搜索
for (int step = 1; step < 5; step++) {
int nr = r + d[0] * step, nc = c + d[1] * step;
if (nr < 0 || nr >= BOARD_SIZE || nc < 0 || nc >= BOARD_SIZE) break;
if (grid[nr][nc] != player) break;
count++;
}
// 反向搜索
for (int step = 1; step < 5; step++) {
int nr = r - d[0] * step, nc = c - d[1] * step;
if (nr < 0 || nr >= BOARD_SIZE || nc < 0 || nc >= BOARD_SIZE) break;
if (grid[nr][nc] != player) break;
count++;
}
if (count >= 5) return true;
}
return false;
}
/**
* 判断终局
*/
int getWinner() {
if (moveHistory.isEmpty()) return 0;
int[] last = moveHistory.get(moveHistory.size() - 1);
int lastPlayer = 3 - currentPlayer;
if (checkWin(last[0], last[1], lastPlayer)) return lastPlayer;
if (moveHistory.size() >= BOARD_SIZE * BOARD_SIZE) return -1; // 平局
return 0;
}
/**
* 转换为神经网络输入（4通道）
*/
double[][][] toTensor() {
double[][][] tensor = new double[4][BOARD_SIZE][BOARD_SIZE];
for (int r = 0; r < BOARD_SIZE; r++) {
for (int c = 0; c < BOARD_SIZE; c++) {
tensor[0][r][c] = (grid[r][c] == currentPlayer) ? 1.0 : 0.0;
tensor[1][r][c] = (grid[r][c] == 3 - currentPlayer) ? 1.0 : 0.0;
tensor[2][r][c] = (grid[r][c] == EMPTY) ? 1.0 : 0.0;
}
}
// 历史热度通道（最近落子位置热度高）
for (int i = 0; i < moveHistory.size(); i++) {
int[] m = moveHistory.get(i);
double heat = (i + 1.0) / moveHistory.size();
tensor[3][m[0]][m[1]] = heat;
}
return tensor;
}
}
/**
* 简化的Actor-Critic网络
*/
static class ActorCriticNetwork {
private final int inputChannels;
private final int boardSize;
private final int hiddenSize;
// 共享卷积层权重（简化为全连接）
double[][] convWeights;
double[] convBias;
// Actor头
double[][] actorWeights;
double[] actorBias;
// Critic头
double[][] criticWeights;
double[] criticBias;
Random rand = new Random(42);
ActorCriticNetwork(int channels, int size, int hidden) {
this.inputChannels = channels;
this.boardSize = size;
this.hiddenSize = hidden;
int inputSize = channels * size * size;
this.convWeights = initMatrix(inputSize, hidden);
this.convBias = new double[hidden];
this.actorWeights = initMatrix(hidden, size * size);
this.actorBias = new double[size * size];
this.criticWeights = initMatrix(hidden, 1);
this.criticBias = new double[1];
}
double[][] initMatrix(int rows, int cols) {
double[][] m = new double[rows][cols];
double scale = Math.sqrt(2.0 / rows);
for (int i = 0; i < rows; i++) {
for (int j = 0; j < cols; j++) {
m[i][j] = rand.nextGaussian() * scale;
}
}
return m;
}
/**
* 前向传播
*/
NetworkOutput forward(double[][][] input) {
// 展平输入
double[] flat = new double[inputChannels * boardSize * boardSize];
int idx = 0;
for (int c = 0; c < inputChannels; c++) {
for (int r = 0; r < boardSize; r++) {
for (int j = 0; j < boardSize; j++) {
flat[idx++] = input[c][r][j];
}
}
}
// 共享层 (ReLU)
double[] shared = new double[hiddenSize];
for (int i = 0; i < hiddenSize; i++) {
double sum = convBias[i];
for (int j = 0; j < flat.length; j++) {
sum += flat[j] * convWeights[j][i];
}
shared[i] = Math.max(0, sum);
}
// Actor输出 (logits)
double[] actorLogits = new double[boardSize * boardSize];
for (int i = 0; i < boardSize * boardSize; i++) {
double sum = actorBias[i];
for (int j = 0; j < hiddenSize; j++) {
sum += shared[j] * actorWeights[j][i];
}
actorLogits[i] = sum;
}
// Critic输出 (value)
double value = criticBias[0];
for (int j = 0; j < hiddenSize; j++) {
value += shared[j] * criticWeights[j][0];
}
return new NetworkOutput(actorLogits, value, shared);
}
/**
* 计算策略概率（带合法动作mask）
*/
double[] getPolicyProbs(double[][][] input, List<int[]> legalMoves) {
NetworkOutput out = forward(input);
double[] logits = out.actorLogits;
// mask非法动作
double[] masked = new double[logits.length];
Arrays.fill(masked, -1e9);
for (int[] m : legalMoves) {
int idx = m[0] * boardSize + m[1];
masked[idx] = logits[idx];
}
// softmax
double max = Double.NEGATIVE_INFINITY;
for (double v : masked) if (v > -1e8) max = Math.max(max, v);
double sum = 0;
double[] exp = new double[masked.length];
for (int i = 0; i < masked.length; i++) {
if (masked[i] > -1e8) {
exp[i] = Math.exp(masked[i] - max);
sum += exp[i];
}
}
for (int i = 0; i < exp.length; i++) {
if (masked[i] > -1e8) exp[i] /= sum;
else exp[i] = 0;
}
return exp;
}
}
static class NetworkOutput {
double[] actorLogits;
double value;
double[] shared;
NetworkOutput(double[] logits, double val, double[] shr) {
this.actorLogits = logits;
this.value = val;
this.shared = shr;
}
}
/**
* 经验元组
*/
static class Experience {
double[][][] state;
int action;
double reward;
double value;
double logProb;
boolean done;
Experience(double[][][] s, int a, double r, double v, double lp, boolean d) {
this.state = deepCopy(s);
this.action = a;
this.reward = r;
this.value = v;
this.logProb = lp;
this.done = d;
}
double[][][] deepCopy(double[][][] src) {
double[][][] dst = new double[src.length][src[0].length][src[0][0].length];
for (int i = 0; i < src.length; i++) {
for (int j = 0; j < src[i].length; j++) {
System.arraycopy(src[i][j], 0, dst[i][j], 0, src[i][j].length);
}
}
return dst;
}
}
/**
* 简化的SGD优化器
*/
static class Optimizer {
double lr;
Optimizer(double lr) { this.lr = lr; }
void step(double[][] weights, double[][] grads) {
for (int i = 0; i < weights.length; i++) {
for (int j = 0; j < weights[i].length; j++) {
weights[i][j] -= lr * grads[i][j];
}
}
}
void step(double[] bias, double[] grads) {
for (int i = 0; i < bias.length; i++) {
bias[i] -= lr * grads[i];
}
}
}
/**
* 自我对弈一局
*/
List<Experience> selfPlay() {
List<Experience> trajectory = new ArrayList<>();
Board board = new Board();
while (true) {
List<int[]> legalMoves = board.getLegalMoves();
if (legalMoves.isEmpty()) break;
double[][][] state = board.toTensor();
double[] probs = network.getPolicyProbs(state, legalMoves);
double value = network.forward(state).value;
// 采样动作
double rand = Math.random();
double cumsum = 0;
int action = legalMoves.get(0)[0] * BOARD_SIZE + legalMoves.get(0)[1];
for (int[] m : legalMoves) {
int idx = m[0] * BOARD_SIZE + m[1];
cumsum += probs[idx];
if (cumsum >= rand) {
action = idx;
break;
}
}
double logProb = Math.log(probs[action] + 1e-10);
int ar = action / BOARD_SIZE, ac = action % BOARD_SIZE;
board.makeMove(ar, ac);
int winner = board.getWinner();
boolean done = winner != 0;
double reward = 0;
if (done) {
if (winner == -1) reward = 0; // 平局
else reward = (winner == 3 - board.currentPlayer) ? 1.0 : -1.0;
}
trajectory.add(new Experience(state, action, reward, value, logProb, done));
if (done) break;
}
return trajectory;
}
/**
* 计算折扣回报
*/
double[] computeReturns(List<Experience> traj) {
double[] returns = new double[traj.size()];
double R = 0;
for (int i = traj.size() - 1; i >= 0; i--) {
R = traj.get(i).reward + GAMMA * R * (traj.get(i).done ? 0 : 1);
returns[i] = R;
}
return returns;
}
/**
* 训练一轮
*/
void trainIteration() {
List<List<Experience>> batch = new ArrayList<>();
// 收集自我对弈数据
for (int g = 0; g < SELF_PLAY_GAMES; g++) {
batch.add(selfPlay());
}
// 计算回报并更新网络
for (List<Experience> traj : batch) {
double[] returns = computeReturns(traj);
for (int i = 0; i < traj.size(); i++) {
Experience exp = traj.get(i);
double advantage = returns[i] - exp.value;
// Actor损失: -log_prob * advantage
double actorLoss = -exp.logProb * advantage;
// Critic损失: (return - value)^2
double criticLoss = (returns[i] - exp.value) * (returns[i] - exp.value);
// 实际实现中需要反向传播计算梯度并更新参数
// 这里展示训练框架的核心逻辑
}
}
System.out.println("Completed training iteration with " + batch.size() + " games");
}
/**
* 使用训练好的AI进行对局
*/
int[] getBestMove(Board board) {
double[][][] state = board.toTensor();
List<int[]> legalMoves = board.getLegalMoves();
double[] probs = network.getPolicyProbs(state, legalMoves);
// 选择概率最高的动作
int bestAction = -1;
double bestProb = -1;
for (int[] m : legalMoves) {
int idx = m[0] * BOARD_SIZE + m[1];
if (probs[idx] > bestProb) {
bestProb = probs[idx];
bestAction = idx;
}
}
return new int[]{bestAction / BOARD_SIZE, bestAction % BOARD_SIZE};
}
public static void main(String[] args) {
GomokuActorCritic agent = new GomokuActorCritic();
// 训练
for (int iter = 0; iter < 100; iter++) {
agent.trainIteration();
}
// 测试对局
Board testBoard = new Board();
for (int turn = 0; turn < 50; turn++) {
int[] move = agent.getBestMove(testBoard);
testBoard.makeMove(move[0], move[1]);
int winner = testBoard.getWinner();
if (winner != 0) {
System.out.println("Winner: " + (winner == BLACK ? "Black" : "White"));
break;
}
}
}
}
四、Actor-Critic训练要点
优势估计是训练稳定性的关键。本文采用蒙特卡洛回报减去Critic估计值作为优势：A_t = R_t - V(s_t)。更高级的实现可以使用GAE（Generalized Advantage Estimation）在偏差和方差之间取得平衡。
熵正则化鼓励策略保持探索性，防止过早收敛到局部最优。损失函数中加入 -β * Σ π(a|s) log π(a|s) 项，其中β为熵系数。
自我对弈的数据质量直接决定训练效果。随着AI水平提升，需要动态调整对手强度（如使用历史版本模型作为对手），避免训练过程中的策略振荡。
五、复杂度分析
网络前向传播的时间复杂度为 O(C·H·W·H_hidden + H_hidden·H·W)，其中C=4为输入通道数，H=W=15为棋盘尺寸。自我对弈一局约产生50-225个经验元组。训练时每轮需要处理 games × avg_moves 条数据，在简化全连接网络上可在CPU上运行，实际生产环境建议使用GPU加速。
六、扩展方向
Actor-Critic可以与其他技术结合进一步提升棋力：引入MCTS进行在线搜索（如AlphaZero框架），在自我对弈中结合神经网络评估和树搜索选择落子。此外，使用ResNet或Transformer替代全连接网络可以更好地捕捉棋盘的空间结构。对于五子棋特有的必胜/必败模式（如冲四、活三），可以在网络输出层加入规则先验，加速收敛。
