引言
在自然语言处理领域，如何让计算机理解词语的语义关系是一个核心挑战。传统的独热编码（One-Hot Encoding）将每个词表示为高维稀疏向量，无法捕捉词与词之间的语义相似性。Word2Vec是Google于2013年提出的词嵌入模型，它通过神经网络将词语映射到低维稠密向量空间，使得语义相近的词语在向量空间中距离更近。本文将用Java实现Word2Vec的Skip-Gram模型，配合负采样（Negative Sampling）优化技术，训练词向量并展示其神奇的类比推理能力。
算法原理
Skip-Gram模型
Skip-Gram的核心思想是：给定一个中心词，预测它周围的上下文词。例如对于句子”猫 喜欢 吃 鱼”，当中心词是”吃”时，Skip-Gram尝试预测”喜欢”和”鱼”。
目标函数为最大化上下文词出现的概率：
$$J = \frac{1}{T} \sum_{t=1}^{T} \sum_{-c \leq j \leq c, j \neq 0} \log P(w_{t+j} | w_t)$$
其中条件概率使用softmax定义：
$$P(w_O | w_I) = \frac{\exp(v_{w_O}’^T \cdot v_{w_I})}{\sum_{w=1}^{W} \exp(v_w’^T \cdot v_{w_I})}$$
负采样优化
原始softmax的分母需要遍历整个词表，计算量巨大。负采样将问题转化为二分类：给定中心词，判断上下文词是否真实共现。对于每个正样本（真实上下文词），随机采样k个负样本（非上下文词）进行训练。
负采样的目标函数：
$$\log \sigma(v_{w_O}’^T \cdot v_{w_I}) + \sum_{i=1}^{k} \mathbb{E}{w_i \sim P_n(w)}[\log \sigma(-v{w_i}’^T \cdot v_{w_I})]$$
词向量训练
每个词有两个向量表示：
– 输入向量（Input Vector）：作为中心词时的向量
– 输出向量（Output Vector）：作为上下文词时的向量
训练完成后，通常使用输入向量作为最终的词嵌入。
Java实现
import java.util.*;
/**
* Word2Vec Skip-Gram with Negative Sampling 实现
*
* 训练词向量，使语义相近的词在向量空间中距离更近
*/
public class Word2Vec {
private int vectorSize; // 词向量维度
private int windowSize; // 上下文窗口大小
private int negativeSamples; // 负采样数量
private double learningRate; // 学习率
private int minCount; // 最小词频阈值
// 词汇表
private Map<String, Integer> wordToId;
private Map<Integer, String> idToWord;
private int[] wordCounts;
private int vocabSize;
// 词向量
private double[][] inputVectors; // 中心词向量
private double[][] outputVectors; // 上下文词向量
// 负采样概率分布表
private double[] negSamplingTable;
private int tableSize = 100000000; // 1亿
public Word2Vec(int vectorSize, int windowSize, int negativeSamples,
double learningRate, int minCount) {
this.vectorSize = vectorSize;
this.windowSize = windowSize;
this.negativeSamples = negativeSamples;
this.learningRate = learningRate;
this.minCount = minCount;
}
/**
* 构建词汇表
*/
public void buildVocab(List<String[]> sentences) {
Map<String, Integer> countMap = new HashMap<>();
for (String[] sentence : sentences) {
for (String word : sentence) {
word = word.toLowerCase();
countMap.put(word, countMap.getOrDefault(word, 0) + 1);
}
}
// 过滤低频词并分配ID
wordToId = new HashMap<>();
idToWord = new HashMap<>();
List<Integer> counts = new ArrayList<>();
int id = 0;
for (Map.Entry<String, Integer> entry : countMap.entrySet()) {
if (entry.getValue() >= minCount) {
wordToId.put(entry.getKey(), id);
idToWord.put(id, entry.getKey());
counts.add(entry.getValue());
id++;
}
}
vocabSize = id;
wordCounts = new int[vocabSize];
for (int i = 0; i < vocabSize; i++) {
wordCounts[i] = counts.get(i);
}
System.out.println("词汇表大小: " + vocabSize);
// 初始化词向量
initVectors();
// 构建负采样表
buildNegSamplingTable();
}
/**
* 初始化词向量
*/
private void initVectors() {
Random rand = new Random(42);
inputVectors = new double[vocabSize][vectorSize];
outputVectors = new double[vocabSize][vectorSize];
for (int i = 0; i < vocabSize; i++) {
for (int j = 0; j < vectorSize; j++) {
inputVectors[i][j] = (rand.nextDouble() - 0.5) / vectorSize;
outputVectors[i][j] = 0.0;
}
}
}
/**
* 构建负采样概率表（基于词频的3/4次幂）
*/
private void buildNegSamplingTable() {
double totalFreq = 0;
for (int count : wordCounts) {
totalFreq += Math.pow(count, 0.75);
}
negSamplingTable = new double[tableSize];
int wordIdx = 0;
double cumProb = Math.pow(wordCounts[0], 0.75) / totalFreq;
for (int i = 0; i < tableSize; i++) {
negSamplingTable[i] = wordIdx;
if ((double) i / tableSize > cumProb && wordIdx < vocabSize - 1) {
wordIdx++;
cumProb += Math.pow(wordCounts[wordIdx], 0.75) / totalFreq;
}
}
}
/**
* 负采样：随机返回一个词ID
*/
private int negativeSampling(Random rand) {
return (int) negSamplingTable[rand.nextInt(tableSize)];
}
/**
* Sigmoid函数
*/
private double sigmoid(double x) {
if (x > 6) return 1.0;
if (x < -6) return 0.0;
return 1.0 / (1.0 + Math.exp(-x));
}
/**
* 计算两个向量的点积
*/
private double dotProduct(double[] a, double[] b) {
double sum = 0;
for (int i = 0; i < a.length; i++) {
sum += a[i] * b[i];
}
return sum;
}
/**
* 训练一个句子
*/
private void trainSentence(String[] sentence, Random rand) {
// 将词转换为ID
List<Integer> wordIds = new ArrayList<>();
for (String word : sentence) {
Integer id = wordToId.get(word.toLowerCase());
if (id != null) {
wordIds.add(id);
}
}
// Skip-Gram：中心词预测上下文词
for (int pos = 0; pos < wordIds.size(); pos++) {
int centerWord = wordIds.get(pos);
// 确定窗口范围
int start = Math.max(0, pos - windowSize);
int end = Math.min(wordIds.size(), pos + windowSize + 1);
for (int ctxPos = start; ctxPos < end; ctxPos++) {
if (ctxPos == pos) continue;
int contextWord = wordIds.get(ctxPos);
// 正样本更新
updatePair(centerWord, contextWord, 1.0);
// 负样本更新
for (int n = 0; n < negativeSamples; n++) {
int negWord = negativeSampling(rand);
if (negWord == contextWord) continue;
updatePair(centerWord, negWord, 0.0);
}
}
}
}
/**
* 更新一对词（中心词，上下文词）
* label=1表示正样本，label=0表示负样本
*/
private void updatePair(int centerWord, int contextWord, double label) {
double[] centerVec = inputVectors[centerWord];
double[] contextVec = outputVectors[contextWord];
// 计算预测值
double score = sigmoid(dotProduct(centerVec, contextVec));
double gradient = (label - score) * learningRate;
// 更新向量
double[] centerGrad = new double[vectorSize];
for (int i = 0; i < vectorSize; i++) {
centerGrad[i] = gradient * contextVec[i];
}
for (int i = 0; i < vectorSize; i++) {
contextVec[i] += gradient * centerVec[i];
centerVec[i] += centerGrad[i];
}
}
/**
* 训练模型
*/
public void train(List<String[]> sentences, int epochs) {
System.out.println("开始训练Word2Vec...");
Random rand = new Random(123);
for (int epoch = 0; epoch < epochs; epoch++) {
// 打乱句子顺序
List<String[]> shuffled = new ArrayList<>(sentences);
Collections.shuffle(shuffled, rand);
for (String[] sentence : shuffled) {
trainSentence(sentence, rand);
}
// 学习率衰减
learningRate *= 0.95;
if ((epoch + 1) % 10 == 0) {
System.out.println("Epoch " + (epoch + 1) + "/" + epochs + " 完成");
}
}
}
/**
* 获取词向量
*/
public double[] getVector(String word) {
Integer id = wordToId.get(word.toLowerCase());
if (id == null) return null;
return inputVectors[id].clone();
}
/**
* 计算两个词的余弦相似度
*/
public double cosineSimilarity(String word1, String word2) {
double[] v1 = getVector(word1);
double[] v2 = getVector(word2);
if (v1 == null || v2 == null) return -1;
double dot = 0, norm1 = 0, norm2 = 0;
for (int i = 0; i < vectorSize; i++) {
dot += v1[i] * v2[i];
norm1 += v1[i] * v1[i];
norm2 += v2[i] * v2[i];
}
return dot / (Math.sqrt(norm1) * Math.sqrt(norm2));
}
/**
* 寻找最相似的词
*/
public List<Map.Entry<String, Double>> mostSimilar(String word, int topN) {
double[] targetVec = getVector(word);
if (targetVec == null) return new ArrayList<>();
Map<String, Double> similarities = new HashMap<>();
for (int i = 0; i < vocabSize; i++) {
String w = idToWord.get(i);
if (w.equalsIgnoreCase(word)) continue;
double[] vec = inputVectors[i];
double dot = 0, norm1 = 0, norm2 = 0;
for (int j = 0; j < vectorSize; j++) {
dot += targetVec[j] * vec[j];
norm1 += targetVec[j] * targetVec[j];
norm2 += vec[j] * vec[j];
}
double sim = dot / (Math.sqrt(norm1) * Math.sqrt(norm2));
similarities.put(w, sim);
}
List<Map.Entry<String, Double>> sorted = new ArrayList<>(similarities.entrySet());
sorted.sort((a, b) -> Double.compare(b.getValue(), a.getValue()));
return sorted.subList(0, Math.min(topN, sorted.size()));
}
/**
* 词向量类比：king - man + woman ≈ queen
*/
public String analogy(String word1, String word2, String word3) {
double[] v1 = getVector(word1);
double[] v2 = getVector(word2);
double[] v3 = getVector(word3);
if (v1 == null || v2 == null || v3 == null) return null;
double[] target = new double[vectorSize];
for (int i = 0; i < vectorSize; i++) {
target[i] = v1[i] - v2[i] + v3[i];
}
// 找最相似的词
String bestWord = null;
double bestSim = -1;
for (int i = 0; i < vocabSize; i++) {
String w = idToWord.get(i);
if (w.equalsIgnoreCase(word1) || w.equalsIgnoreCase(word2) ||
w.equalsIgnoreCase(word3)) continue;
double[] vec = inputVectors[i];
double dot = 0, normT = 0, normV = 0;
for (int j = 0; j < vectorSize; j++) {
dot += target[j] * vec[j];
normT += target[j] * target[j];
normV += vec[j] * vec[j];
}
double sim = dot / (Math.sqrt(normT) * Math.sqrt(normV));
if (sim > bestSim) {
bestSim = sim;
bestWord = w;
}
}
return bestWord;
}
public static void main(String[] args) {
// 构建演示语料（小型语义域：颜色、水果、动物）
List<String[]> corpus = new ArrayList<>();
String[][] demoSentences = {
{"red", "apple", "is", "sweet"},
{"green", "apple", "is", "sour"},
{"yellow", "banana", "is", "sweet"},
{"red", "cherry", "is", "sweet"},
{"cat", "eats", "fish"},
{"dog", "eats", "meat"},
{"cat", "is", "small"},
{"dog", "is", "big"},
{"apple", "is", "fruit"},
{"banana", "is", "fruit"},
{"cherry", "is", "fruit"},
{"cat", "is", "animal"},
{"dog", "is", "animal"},
{"red", "is", "color"},
{"green", "is", "color"},
{"yellow", "is", "color"},
{"the", "red", "apple"},
{"the", "green", "apple"},
{"the", "yellow", "banana"},
{"a", "cat", "and", "a", "dog"},
{"big", "dog", "small", "cat"},
{"sweet", "fruit", "sour", "apple"},
{"fish", "for", "cat", "meat", "for", "dog"},
{"color", "of", "apple", "is", "red"},
{"color", "of", "banana", "is", "yellow"},
{"color", "of", "cherry", "is", "red"}
};
corpus.addAll(Arrays.asList(demoSentences));
// 创建并训练模型
Word2Vec model = new Word2Vec(50, 2, 5, 0.025, 1);
model.buildVocab(corpus);
model.train(corpus, 1000);
// 测试相似度
System.out.println("\n=== 词语相似度测试 ===");
System.out.printf("apple vs banana: %.4f%n", model.cosineSimilarity("apple", "banana"));
System.out.printf("apple vs cat: %.4f%n", model.cosineSimilarity("apple", "cat"));
System.out.printf("cat vs dog: %.4f%n", model.cosineSimilarity("cat", "dog"));
System.out.printf("red vs green: %.4f%n", model.cosineSimilarity("red", "green"));
// 找最相似的词
System.out.println("\n=== 最相似词 ===");
System.out.println("与 'apple' 最相似的词:");
for (Map.Entry<String, Double> e : model.mostSimilar("apple", 5)) {
System.out.printf(" %s: %.4f%n", e.getKey(), e.getValue());
}
// 词向量类比
System.out.println("\n=== 词向量类比 ===");
System.out.println("cat : animal ≈ apple : ?");
String result = model.analogy("cat", "animal", "apple");
System.out.println("预测结果: " + result);
}
}
代码解析
词汇表构建
代码首先统计语料中每个词的频率，过滤掉低频词，为每个词分配唯一ID。词汇表是后续所有操作的基础。
负采样表
负采样表是一个基于词频3/4次幂的概率分布。高频词（如”the”）虽然出现次数多，但经过3/4次幂压缩后，其被采样的概率比线性比例更均衡。
Skip-Gram训练
对于每个中心词，代码遍历其窗口内的所有上下文词作为正样本，同时随机采样k个负样本。每个样本对（中心词，上下文词）都通过sigmoid进行二分类，并更新对应的词向量。
词向量类比
Word2Vec最著名的特性是词向量支持线性类比。代码中通过向量运算 v(word1) - v(word2) + v(word3) 来寻找类比关系。
复杂度分析
| 指标 | 复杂度 | 说明 |
|---|---|---|
| 训练时间 | O(E × T × W × (2K+1) × D) | E为轮数，T为token数，W为窗口，K为负采样数，D为维度 |
| 空间复杂度 | O(V × D) | V为词汇表大小 |
| 查相似度 | O(V × D) | 需与所有词计算余弦距离 |
Word2Vec的演进
- CBOW模型：与Skip-Gram相反，用上下文词预测中心词，训练更快
- GloVe：结合全局统计信息和局部上下文，效果通常更好
- FastText：引入子词（subword）信息，解决OOV问题
- BERT/GPT：基于Transformer的上下文相关词向量，一个多义词在不同上下文中有不同表示
总结
本文通过Java实现了Word2Vec的Skip-Gram负采样模型，展示了如何将词语映射到稠密向量空间。词向量的神奇之处在于，简单的向量运算就能捕捉复杂的语义关系——这正是神经网络从海量文本中自动学习语言规律的结果。理解Word2Vec是进入现代自然语言处理的重要一步。
