[
https://issues.apache.org/jira/browse/OPENNLP-1991?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18126307#comment-18126307
]
ASF GitHub Bot commented on OPENNLP-1991:
-----------------------------------------
krickert commented on code in PR #1323:
URL: https://github.com/apache/opennlp/pull/1323#discussion_r4247892306
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java:
##########
@@ -0,0 +1,942 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.ParamChecks;
+import opennlp.tools.util.StringUtil;
+
+/**
+ * The weights of the feedforward transition parser: embeddings for words,
tags, and arc
+ * labels, one hidden layer with cube activation, and a transition output
layer, stored
+ * in a versioned binary format. The architecture follows
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>The same class scores configurations during training and decoding.
Unknown words
+ * use a learned fallback row; words are matched case-insensitively after
+ * {@link #normalize(String) normalization}.</p>
+ *
+ * <p>Instances returned by {@link FeedforwardDependencyTrainer} are immutable
and safe
Review Comment:
Reworded to the actual contract: the weights are not modified after
construction or loading and a model can be shared between threads and parsers.
The parser no longer touches the model; the optional cache lives in the parser.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java:
##########
@@ -0,0 +1,1176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Function;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ParamChecks;
+
+/**
+ * Trains the {@link FeedforwardDependencyModel}: oracle-derived transition
examples,
+ * minibatch AdaGrad over a softmax cross-entropy loss, cube activation, and
inverted
+ * dropout on the hidden layer, the training recipe of
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>Words below the frequency cutoff share a learned unknown embedding;
absent
+ * template positions share a learned padding embedding. Non-projective
samples have no
+ * arc-standard derivation and are skipped. Training is deterministic for a
fixed
+ * {@link Settings#seed()}.</p>
+ *
+ * @since 3.0.0
+ */
+public final class FeedforwardDependencyTrainer {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(FeedforwardDependencyTrainer.class);
+
+ /** The AdaGrad denominator offset that keeps the first steps finite. */
+ private static final double ADAGRAD_EPSILON = 1e-6;
+
+ /** The derivative factor of the cube activation, {@code d/dx x^3 = 3x^2}. */
+ private static final double CUBE_DERIVATIVE_FACTOR = 3.0;
+
+ /** The numerator of the Glorot uniform bound, {@code sqrt(6 / (fanIn +
fanOut))}. */
+ private static final double GLOROT_NUMERATOR = 6.0;
+
+ /** The half-width of the uniform range embeddings are drawn from. */
+ private static final double EMBEDDING_INIT_SCALE = 0.01;
+
+ /** The smallest probability the reported training loss takes the log of. */
+ private static final double LOSS_PROBABILITY_FLOOR = 1e-12;
+
+ /** The sentence loss reported when refinement made no update for a
sentence. */
+ private static final double NO_UPDATE = -1.0;
+
+ /** Special-symbol rows present before any training vocabulary is added. */
+ private static final int MIN_VOCABULARY_ROWS = 8;
+
+ /** SHIFT and at least one RIGHT_ARC are required to parse a sentence. */
+ private static final int MIN_TRANSITIONS = 2;
+
+ /** Prevents construction of this utility class. */
+ private FeedforwardDependencyTrainer() {
+ }
+
+ /**
+ * The training hyperparameters.
+ *
+ * @param embeddingSize The embedding dimensionality. Must be positive.
+ * @param hiddenSize The hidden layer width. Must be positive.
+ * @param epochs The number of passes over the examples. Must be positive.
+ * @param batchSize The minibatch size. Must be positive.
+ * @param learningRate The AdaGrad step size. Must be positive.
+ * @param l2 The L2 penalty applied to the dense weights. Must not be
negative.
+ * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}.
+ * @param wordCutoff The minimum frequency for a word to get its own
embedding. Must
+ * not be negative.
+ * @param seed The random seed making a run reproducible.
+ */
+ public record Settings(int embeddingSize, int hiddenSize, int epochs, int
batchSize,
+ double learningRate, double l2, double dropout, int wordCutoff, long
seed) {
+
+ /** The default embedding dimensionality. */
+ private static final int DEFAULT_EMBEDDING_SIZE = 50;
+
+ /** The default hidden layer width. */
+ private static final int DEFAULT_HIDDEN_SIZE = 200;
+
+ /** The default number of epochs. */
+ private static final int DEFAULT_EPOCHS = 10;
+
+ /** The default minibatch size. */
+ private static final int DEFAULT_BATCH_SIZE = 256;
+
+ /** The default AdaGrad step size. */
+ private static final double DEFAULT_LEARNING_RATE = 0.02;
+
+ /** The default L2 penalty. */
+ private static final double DEFAULT_L2 = 1e-8;
+
+ /** The default hidden dropout probability. */
+ private static final double DEFAULT_DROPOUT = 0.5;
+
+ /** The default minimum word frequency for an embedding row. */
+ private static final int DEFAULT_WORD_CUTOFF = 2;
+
+ /** The default random seed. */
+ static final long DEFAULT_SEED = 17L;
+
+ /**
+ * Validates the hyperparameters.
+ *
+ * @throws IllegalArgumentException Thrown if a value is outside its
documented
+ * range.
+ */
+ public Settings {
+ requirePositive(embeddingSize, "embeddingSize");
+ requirePositive(hiddenSize, "hiddenSize");
+ requirePositive(epochs, "epochs");
+ requirePositive(batchSize, "batchSize");
+ if (embeddingSize > FeedforwardDependencyModel.MAX_EMBEDDING_SIZE) {
+ throw new IllegalArgumentException("embeddingSize exceeds the model
format limit: "
+ + embeddingSize);
+ }
+ if (hiddenSize > FeedforwardDependencyModel.MAX_HIDDEN_SIZE) {
+ throw new IllegalArgumentException("hiddenSize exceeds the model
format limit: "
+ + hiddenSize);
+ }
+ final long minimumModelValues = modelFloatValues(MIN_VOCABULARY_ROWS,
+ MIN_TRANSITIONS, embeddingSize, hiddenSize);
+ if (minimumModelValues >
FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) {
+ throw new IllegalArgumentException("embeddingSize and hiddenSize
exceed the model "
+ + "format allocation limit");
+ }
+ if (ParamChecks.requireFinite(learningRate, "learningRate") <= 0.0) {
+ throw new IllegalArgumentException("learningRate must be positive: " +
learningRate);
+ }
+ ParamChecks.requireNonNegative(ParamChecks.requireFinite(l2, "l2"),
"l2");
+ if (!(dropout >= 0.0 && dropout < 1.0)) {
+ throw new IllegalArgumentException("dropout must be in [0, 1): " +
dropout);
+ }
+ ParamChecks.requireNonNegative(wordCutoff, "wordCutoff");
+ }
+
+ /**
+ * Rejects a hyperparameter that is not positive.
+ *
+ * @param value The value to check.
+ * @param name The hyperparameter name used in the exception message.
+ * @throws IllegalArgumentException Thrown if {@code value} is not
positive.
+ */
+ private static void requirePositive(int value, String name) {
+ if (value <= 0) {
+ throw new IllegalArgumentException(name + " must be positive: " +
value);
+ }
+ }
+
+ /**
+ * @return The default hyperparameters. Never {@code null}.
+ */
+ public static Settings defaults() {
+ return new Settings(DEFAULT_EMBEDDING_SIZE, DEFAULT_HIDDEN_SIZE,
DEFAULT_EPOCHS,
+ DEFAULT_BATCH_SIZE, DEFAULT_LEARNING_RATE, DEFAULT_L2,
DEFAULT_DROPOUT,
+ DEFAULT_WORD_CUTOFF, DEFAULT_SEED);
+ }
+ }
+
+ /**
+ * Trains a model from dependency samples.
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null} or
no
+ * trainable example can be derived from the samples.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings) throws IOException {
+ return train(samples, settings, null);
+ }
+
+ /**
+ * Trains a model from dependency samples, seeding word embeddings from a
pretrained
+ * source.
+ *
+ * <p>The provider is consulted once per vocabulary word during
initialization; words
+ * it returns {@code null} for keep their random initialization, and all
embeddings
+ * remain trainable afterwards. The pretrained source is a training-time
ingredient
+ * only: the learned embeddings ship inside the model, so parsing carries no
+ * dependency on the source.</p>
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @param pretrained Maps a normalized word to its pretrained vector of
exactly
+ * {@link Settings#embeddingSize()} dimensions, or {@code
null} for
+ * unknown words. May be {@code null} to disable seeding.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if {@code samples} or {@code
settings} is
+ * {@code null}, no trainable example can be derived, or a
pretrained vector
+ * has the wrong dimensionality.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings, Function<String, float[]> pretrained)
+ throws IOException {
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ final List<DependencySample> corpus = readAll(samples);
+ final FeedforwardDependencyModel model = initialize(corpus, settings);
+ if (pretrained != null) {
+ seed(model, pretrained, settings);
+ }
+ final List<int[]> featureList = new ArrayList<>();
+ final List<Integer> goldList = new ArrayList<>();
+ collectExamples(corpus, model, featureList, goldList);
+ if (featureList.isEmpty()) {
+ throw new IllegalArgumentException("no trainable examples in the
samples");
+ }
+ optimize(model, featureList, goldList, settings);
+ return model;
+ }
+
+ /**
+ * Refines a locally trained model with beam search and the early-update
method of
+ * <a href="https://aclanthology.org/P04-1015/";>Collins and Roark
(2004)</a>. The loss
+ * is conditional likelihood over candidate paths scored by summed
log-probabilities.
+ *
+ * <p>Refinement updates a copy with per-sentence AdaGrad steps and no
dropout.
+ * {@link Settings#epochs()} sets the number of refinement passes. Use the
same beam
+ * size for parsing the result.</p>
+ *
+ * <p>The transition inventory comes from {@code model} and is not extended,
because
+ * its size is the width of the trained output layer. A refinement corpus
using a
+ * relation label the original training set lacked is therefore rejected
rather than
+ * silently ignored.</p>
+ *
+ * @param model The locally trained model to refine. Left untouched. Must
not be
+ * {@code null}.
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters; {@code epochs}, {@code learningRate},
+ * {@code l2}, and {@code seed} apply. Must not be {@code
null}.
+ * @param beamSize The beam width to track the gold derivation in. Must be
at least 2.
+ * @return A new refined model, distinct from {@code model}. Never {@code
null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null},
+ * {@code beamSize} is below 2, no trainable sample can be derived,
or a sample
+ * requires a transition {@code model} does not know.
+ */
+ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel
model,
+ ObjectStream<DependencySample> samples, Settings settings, int beamSize)
+ throws IOException {
+ ParamChecks.requireNonNullArg(model, "model");
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ if (beamSize < 2) {
+ throw new IllegalArgumentException("beamSize must be at least 2: " +
beamSize);
+ }
+ final List<DependencySample> corpus = readAll(samples);
+ final String[] outcomes = model.transitions();
+ final Map<String, Integer> transitionIds = new HashMap<>();
+ final Transition[] transitions = new Transition[outcomes.length];
+ for (int i = 0; i < transitions.length; i++) {
+ transitionIds.put(outcomes[i], i);
+ transitions[i] = Transition.decode(outcomes[i]);
+ }
+
+ final List<DependencySample> trainable = new ArrayList<>();
+ final List<int[]> oracles = new ArrayList<>();
+ for (final DependencySample s : corpus) {
+ if (!ArcStandardOracle.isProjective(s.getTree())) {
+ continue;
+ }
+ final List<Transition> oracle =
ArcStandardOracle.transitions(s.getTree());
+ final int[] encoded = new int[oracle.size()];
+ for (int i = 0; i < encoded.length; i++) {
+ final String outcome = oracle.get(i).encode();
+ final Integer id = transitionIds.get(outcome);
+ if (id == null) {
+ // The outcome space was fixed by the original training set, so a
relation
+ // label it never saw has no output unit to push probability onto.
+ throw new IllegalArgumentException(
+ "unknown transition in the refinement samples: " + outcome);
+ }
+ encoded[i] = id;
+ }
+ trainable.add(s);
+ oracles.add(encoded);
+ }
+ if (trainable.isEmpty()) {
+ throw new IllegalArgumentException("no trainable samples for
refinement");
+ }
+
+ final FeedforwardDependencyModel refined = model.copy();
+ final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings);
+ final Random random = new Random(settings.seed());
+ final int[] order = new int[trainable.size()];
+ for (int i = 0; i < order.length; i++) {
+ order[i] = i;
+ }
+ for (int epoch = 1; epoch <= settings.epochs(); epoch++) {
+ final long epochStart = System.currentTimeMillis();
+ shuffle(order, random);
+ double loss = 0.0;
+ int updates = 0;
+ for (final int index : order) {
+ final double sentenceLoss =
optimizer.refineSentence(trainable.get(index),
+ oracles.get(index), transitions, beamSize);
+ if (sentenceLoss >= 0.0) {
+ loss += sentenceLoss;
+ updates++;
+ }
+ }
+ checkFinite(refined);
+ logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch,
+ loss / Math.max(updates, 1), updates, System.currentTimeMillis() -
epochStart);
+ }
+ return refined;
+ }
+
+ /** One candidate path in the refinement beam: the parent link forms the
history. */
+ private static final class BeamNode {
+ private final BeamNode parent;
+ private final int[] features;
+ private final int transition;
+ private final double score;
+ private final boolean gold;
+ private ArcStandardState state;
+
+ /**
+ * Extends {@code parent} by one transition; the start node passes {@code
null}.
+ *
+ * @param parent The node this one extends, or {@code null} for the start
node.
+ * @param features The embedding rows the transition was scored on.
+ * @param transition The transition index taken from {@code parent}.
+ * @param score The summed log-probability of the path.
+ * @param gold Whether the path is the gold derivation so far.
+ */
+ private BeamNode(BeamNode parent, int[] features, int transition, double
score,
+ boolean gold) {
+ this.parent = parent;
+ this.features = features;
+ this.transition = transition;
+ this.score = score;
+ this.gold = gold;
+ }
+ }
+
+ /** The forward, backward, and AdaGrad state for global refinement. */
+ private static final class GlobalOptimizer {
+ private final FeedforwardDependencyModel model;
+ private final Settings settings;
+ private final int embeddingSize;
+ private final int hiddenSize;
+ private final int outputSize;
+ private final int inputSize;
+
+ private final double[][] embeddingAccumulator;
+ private final double[][] hiddenAccumulator;
+ private final double[] hiddenBiasAccumulator;
+ private final double[][] outputAccumulator;
+ private final double[] outputBiasAccumulator;
+
+ private final double[][] hiddenGradient;
+ private final double[] hiddenBiasGradient;
+ private final double[][] outputGradient;
+ private final double[] outputBiasGradient;
+ private final Map<Integer, double[]> embeddingGradients = new HashMap<>();
+
+ private final double[] x;
+ private final double[] pre;
+ private final double[] hidden;
+ private final double[] probabilities;
+ private final double[] hiddenDelta;
+ private final double[] inputDelta;
+
+ /**
+ * Sizes the accumulators and scratch buffers for one refinement run.
+ *
+ * @param model The model copy the run updates in place.
+ * @param settings The hyperparameters; {@code learningRate} and {@code
l2} apply.
+ */
+ private GlobalOptimizer(FeedforwardDependencyModel model, Settings
settings) {
+ this.model = model;
+ this.settings = settings;
+ this.embeddingSize = model.embeddings()[0].length;
+ this.hiddenSize = model.hiddenBias().length;
+ this.outputSize = model.outputBias().length;
+ this.inputSize = FeedforwardContext.FEATURE_COUNT * embeddingSize;
+ this.embeddingAccumulator =
+ new double[model.embeddings().length][embeddingSize];
+ this.hiddenAccumulator = new double[hiddenSize][inputSize];
+ this.hiddenBiasAccumulator = new double[hiddenSize];
+ this.outputAccumulator = new double[outputSize][hiddenSize];
+ this.outputBiasAccumulator = new double[outputSize];
+ this.hiddenGradient = new double[hiddenSize][inputSize];
+ this.hiddenBiasGradient = new double[hiddenSize];
+ this.outputGradient = new double[outputSize][hiddenSize];
+ this.outputBiasGradient = new double[outputSize];
+ this.x = new double[inputSize];
+ this.pre = new double[hiddenSize];
+ this.hidden = new double[hiddenSize];
+ this.probabilities = new double[outputSize];
+ this.hiddenDelta = new double[hiddenSize];
+ this.inputDelta = new double[inputSize];
+ }
+
+ /**
+ * Decodes one sentence with the beam, updating on the early-update point
or the
+ * final beam.
+ *
+ * @param sample The sentence.
+ * @param oracle The gold transition indexes.
+ * @param transitions The decoded transition inventory.
+ * @param beamSize The beam width.
+ * @return The sentence loss, or {@link
FeedforwardDependencyTrainer#NO_UPDATE} when the
+ * sentence produced no update.
+ */
+ private double refineSentence(DependencySample sample, int[] oracle,
+ Transition[] transitions, int beamSize) {
+ final String[] tokens = sample.getTokens();
+ final String[] tags = sample.getTags();
+ final BeamNode root = new BeamNode(null, null, -1, 0.0, true);
+ root.state = new ArcStandardState(tokens.length);
+ List<BeamNode> beam = List.of(root);
+
+ for (int step = 0; step < oracle.length; step++) {
+ final List<BeamNode> expansions = new ArrayList<>();
+ BeamNode goldChild = null;
+ for (final BeamNode node : beam) {
+ final int[] features =
+ model.featureIds(FeedforwardContext.extract(node.state, tokens,
tags));
+ forward(features);
+ logSoftmaxInPlace(probabilities);
+ for (int i = 0; i < outputSize; i++) {
+ if (node.state.canApply(transitions[i])) {
+ final boolean goldNext = node.gold && i == oracle[step];
+ final BeamNode child = new BeamNode(node, features, i,
+ node.score + probabilities[i], goldNext);
+ expansions.add(child);
+ if (goldNext) {
+ goldChild = child;
+ }
+ }
+ }
+ }
+ expansions.sort((a, b) -> Double.compare(b.score, a.score));
+ final List<BeamNode> survivors =
+ new ArrayList<>(expansions.subList(0, Math.min(beamSize,
expansions.size())));
+ boolean goldRetained = false;
+ for (final BeamNode survivor : survivors) {
+ if (survivor.gold) {
+ goldRetained = true;
+ break;
+ }
+ }
+ if (!goldRetained) {
+ if (goldChild == null) {
+ // The gold transition was not applicable in the gold
configuration, so
+ // this sentence yields no update.
+ return NO_UPDATE;
+ }
+ survivors.add(goldChild);
+ return updateFromCandidates(survivors);
+ }
+ if (step == oracle.length - 1) {
+ return updateFromCandidates(survivors);
+ }
+ for (final BeamNode survivor : survivors) {
+ survivor.state = survivor.parent.state.copy();
+ survivor.state.apply(transitions[survivor.transition]);
+ }
+ beam = survivors;
+ }
+ return NO_UPDATE;
+ }
+
+ /**
+ * Applies the conditional-likelihood update over the candidate paths.
+ *
+ * @param candidates The surviving paths plus, on early update, the gold
path.
+ * @return The negative log-probability of the gold path under the
candidates.
+ */
+ private double updateFromCandidates(List<BeamNode> candidates) {
+ double max = Double.NEGATIVE_INFINITY;
+ double goldScore = Double.NEGATIVE_INFINITY;
+ for (final BeamNode candidate : candidates) {
+ max = Math.max(max, candidate.score);
+ if (candidate.gold) {
+ goldScore = candidate.score;
+ }
+ }
+ double normalizer = 0.0;
+ for (final BeamNode candidate : candidates) {
+ normalizer += Math.exp(candidate.score - max);
+ }
+ final double logNormalizer = max + Math.log(normalizer);
+
+ zero(hiddenGradient);
+ Arrays.fill(hiddenBiasGradient, 0.0);
+ zero(outputGradient);
+ Arrays.fill(outputBiasGradient, 0.0);
+ embeddingGradients.clear();
+ for (final BeamNode candidate : candidates) {
+ final double weight = Math.exp(candidate.score - logNormalizer)
+ - (candidate.gold ? 1.0 : 0.0);
+ if (weight == 0.0) {
+ continue;
+ }
+ for (BeamNode node = candidate; node.parent != null; node =
node.parent) {
+ backward(node.features, node.transition, weight);
+ }
+ }
+ update(model.hiddenWeights(), hiddenGradient, hiddenAccumulator, 1,
settings);
+ updateVector(model.hiddenBias(), hiddenBiasGradient,
hiddenBiasAccumulator, 1,
+ settings);
+ update(model.outputWeights(), outputGradient, outputAccumulator, 1,
settings);
+ updateVector(model.outputBias(), outputBiasGradient,
outputBiasAccumulator, 1,
+ settings);
+ for (final Map.Entry<Integer, double[]> entry :
embeddingGradients.entrySet()) {
+ final float[] embeddingRow = model.embeddings()[entry.getKey()];
+ final double[] accumulatorRow = embeddingAccumulator[entry.getKey()];
+ final double[] gradientRow = entry.getValue();
+ for (int d = 0; d < embeddingSize; d++) {
+ final double gradient = gradientRow[d];
+ accumulatorRow[d] += gradient * gradient;
+ embeddingRow[d] -= settings.learningRate() * gradient
+ / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON);
+ }
+ }
+ return logNormalizer - goldScore;
+ }
+
+ /**
+ * Computes hidden activations and raw output scores for one feature
vector into
+ * the scratch buffers.
+ *
+ * @param features The embedding rows of the configuration.
+ */
+ private void forward(int[] features) {
+ final float[][] embeddings = model.embeddings();
+ for (int f = 0; f < features.length; f++) {
+ final float[] embedding = embeddings[features[f]];
+ final int offset = f * embeddingSize;
+ for (int d = 0; d < embeddingSize; d++) {
+ x[offset + d] = embedding[d];
+ }
+ }
+ final float[][] hiddenWeights = model.hiddenWeights();
+ final float[] hiddenBias = model.hiddenBias();
+ for (int j = 0; j < hiddenSize; j++) {
+ final float[] weightRow = hiddenWeights[j];
+ double sum = hiddenBias[j];
+ for (int k = 0; k < inputSize; k++) {
+ sum += weightRow[k] * x[k];
+ }
+ pre[j] = sum;
+ hidden[j] = sum * sum * sum;
+ }
+ final float[][] outputWeights = model.outputWeights();
+ final float[] outputBias = model.outputBias();
+ for (int o = 0; o < outputSize; o++) {
+ final float[] weightRow = outputWeights[o];
+ double sum = outputBias[o];
+ for (int j = 0; j < hiddenSize; j++) {
+ sum += weightRow[j] * hidden[j];
+ }
+ probabilities[o] = sum;
+ }
+ }
+
+ /**
+ * Accumulates gradients for one decoded step: the weighted difference
between the
+ * step's softmax and its chosen transition.
+ *
+ * @param features The step's input features.
+ * @param chosen The transition the path took at this step.
+ * @param weight The path's weight in the candidate distribution.
+ */
+ private void backward(int[] features, int chosen, double weight) {
+ forward(features);
+ double max = Double.NEGATIVE_INFINITY;
+ for (int o = 0; o < outputSize; o++) {
+ max = Math.max(max, probabilities[o]);
+ }
+ double normalizer = 0.0;
+ for (int o = 0; o < outputSize; o++) {
+ probabilities[o] = Math.exp(probabilities[o] - max);
+ normalizer += probabilities[o];
+ }
+ Arrays.fill(hiddenDelta, 0.0);
+ Arrays.fill(inputDelta, 0.0);
+ final float[][] outputWeights = model.outputWeights();
+ for (int o = 0; o < outputSize; o++) {
+ // dL/dlogit for a path's step under the conditional likelihood: the
path weight
+ // times how the step's log-probability responds to this logit
+ final double delta =
+ weight * ((o == chosen ? 1.0 : 0.0) - probabilities[o] /
normalizer);
+ outputBiasGradient[o] += delta;
+ final double[] gradientRow = outputGradient[o];
+ final float[] weightRow = outputWeights[o];
+ for (int j = 0; j < hiddenSize; j++) {
+ gradientRow[j] += delta * hidden[j];
+ hiddenDelta[j] += delta * weightRow[j];
+ }
+ }
+ final float[][] hiddenWeights = model.hiddenWeights();
+ for (int j = 0; j < hiddenSize; j++) {
+ final double preDelta = hiddenDelta[j] * CUBE_DERIVATIVE_FACTOR *
pre[j] * pre[j];
+ hiddenBiasGradient[j] += preDelta;
+ final double[] gradientRow = hiddenGradient[j];
+ final float[] weightRow = hiddenWeights[j];
+ for (int k = 0; k < inputSize; k++) {
+ gradientRow[k] += preDelta * x[k];
+ inputDelta[k] += preDelta * weightRow[k];
+ }
+ }
+ for (int f = 0; f < features.length; f++) {
+ final double[] embeddingGradient = embeddingGradients
+ .computeIfAbsent(features[f], key -> new double[embeddingSize]);
+ final int offset = f * embeddingSize;
+ for (int d = 0; d < embeddingSize; d++) {
+ embeddingGradient[d] += inputDelta[offset + d];
+ }
+ }
+ }
+
+ /**
+ * Turns raw scores into log-probabilities in place.
+ *
+ * @param scores The raw scores, overwritten with their log-softmax.
+ */
+ private void logSoftmaxInPlace(double[] scores) {
+ double max = Double.NEGATIVE_INFINITY;
+ for (final double score : scores) {
+ max = Math.max(max, score);
+ }
+ double sum = 0.0;
+ for (final double score : scores) {
+ sum += Math.exp(score - max);
+ }
+ final double logSum = Math.log(sum);
+ for (int i = 0; i < scores.length; i++) {
+ scores[i] = (scores[i] - max) - logSum;
+ }
+ }
+ }
+
+ /**
+ * Reads a sample stream into memory; both trainers pass over the corpus
repeatedly.
+ *
+ * @param samples The stream to drain.
+ * @return All samples in stream order. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ */
+ private static List<DependencySample> readAll(ObjectStream<DependencySample>
samples)
+ throws IOException {
+ final List<DependencySample> corpus = new ArrayList<>();
+ DependencySample sample;
+ while ((sample = samples.read()) != null) {
+ corpus.add(sample);
+ }
+ return corpus;
+ }
+
+ /**
+ * Overwrites the random word rows with pretrained vectors where available.
+ *
+ * @param model The freshly initialized model whose word rows are seeded.
+ * @param pretrained The pretrained vector source, returning {@code null}
for a word it
+ * does not cover.
+ * @param settings The hyperparameters, which fix the expected vector width.
+ * @throws IllegalArgumentException Thrown if a pretrained vector has a
different width
+ * than the embedding size or contains a non-finite value.
+ */
+ private static void seed(FeedforwardDependencyModel model,
+ Function<String, float[]> pretrained, Settings settings) {
+ int seeded = 0;
+ for (final Map.Entry<String, Integer> entry : model.wordIds().entrySet()) {
+ if (FeedforwardDependencyModel.isSpecialSymbol(entry.getKey())) {
+ // The special unknown, padding, and root symbols have no pretrained
+ // counterpart, so they keep their random initialization.
+ continue;
+ }
+ final float[] vector = pretrained.apply(entry.getKey());
+ if (vector == null) {
+ continue;
+ }
+ if (vector.length != settings.embeddingSize()) {
+ throw new IllegalArgumentException("pretrained vector for '" +
entry.getKey()
+ + "' has " + vector.length + " dimensions, expected "
+ + settings.embeddingSize());
+ }
+ for (final float value : vector) {
+ if (!Float.isFinite(value)) {
+ throw new IllegalArgumentException(
+ "pretrained vector for '" + entry.getKey() + "' contains a
non-finite value");
+ }
+ }
+ System.arraycopy(vector, 0, model.embeddings()[entry.getValue()], 0,
vector.length);
+ seeded++;
+ }
+ logger.info("seeded {} of {} word embeddings from the pretrained source",
seeded,
+ model.wordIds().size());
+ }
+
+ /**
+ * Builds the vocabularies and randomly initialized weights. A word, tag, or
label in
+ * the training data that spells a reserved symbol shares the reserved row,
as an
+ * unknown symbol does, so the reserved rows are never displaced.
+ *
+ * @param corpus The training samples.
+ * @param settings The hyperparameters.
+ * @return The untrained model. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if the vocabularies and settings
would
+ * produce more model values than the model format can store.
+ */
+ private static FeedforwardDependencyModel initialize(List<DependencySample>
corpus,
+ Settings settings) {
+ final Map<String, Integer> wordCounts = new HashMap<>();
+ final Map<String, Integer> tagIds = new HashMap<>();
+ final Map<String, Integer> labelIds = new HashMap<>();
+ final Map<String, Integer> transitionIds = new HashMap<>();
+ for (final DependencySample s : corpus) {
+ if (!ArcStandardOracle.isProjective(s.getTree())) {
+ continue;
+ }
+ final List<Transition> oracle =
ArcStandardOracle.transitions(s.getTree());
+ for (final String token : s.getTokens()) {
+ wordCounts.merge(FeedforwardDependencyModel.normalize(token), 1,
Integer::sum);
+ }
+ for (final String tag : s.getTags()) {
+ tagIds.putIfAbsent(tag, 0);
+ }
+ final DependencyTree tree = s.getTree();
+ for (int i = 0; i < tree.size(); i++) {
+ labelIds.putIfAbsent(tree.relationOf(i), 0);
+ }
+ for (final Transition transition : oracle) {
+ transitionIds.putIfAbsent(transition.encode(), 0);
+ }
+ }
+
+ int row = 0;
+ final Map<String, Integer> wordIds = new HashMap<>();
+ row = addSpecialSymbols(wordIds, row, FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE,
DependencyContextGenerator.ROOT_VALUE);
+ for (final Map.Entry<String, Integer> entry : wordCounts.entrySet()) {
+ if (entry.getValue() >= settings.wordCutoff() &&
!wordIds.containsKey(entry.getKey())) {
+ wordIds.put(entry.getKey(), row++);
+ }
+ }
+ final Map<String, Integer> tags = new HashMap<>();
+ row = addSpecialSymbols(tags, row, FeedforwardDependencyModel.UNKNOWN,
Review Comment:
Documented in the trainer Javadoc and covered by
`testUnknownRowsAreTrainedOnlyForWordsBelowTheCutoff`: the tag and label
unknown rows keep their initialization, the word unknown row trains only when
words fall below the cutoff. I left symbol dropout for a follow-up since it
changes the training recipe.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java:
##########
@@ -0,0 +1,942 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.ParamChecks;
+import opennlp.tools.util.StringUtil;
+
+/**
+ * The weights of the feedforward transition parser: embeddings for words,
tags, and arc
+ * labels, one hidden layer with cube activation, and a transition output
layer, stored
+ * in a versioned binary format. The architecture follows
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>The same class scores configurations during training and decoding.
Unknown words
+ * use a learned fallback row; words are matched case-insensitively after
+ * {@link #normalize(String) normalization}.</p>
+ *
+ * <p>Instances returned by {@link FeedforwardDependencyTrainer} are immutable
and safe
+ * to share between threads. {@link FeedforwardDependencyTrainer#refine
refine} updates
+ * an independent copy.</p>
+ *
+ * @see FeedforwardDependencyParser
+ * @see FeedforwardDependencyTrainer
+ * @since 3.0.0
+ */
+@ThreadSafe
+public class FeedforwardDependencyModel {
+
+ /** The format header written before every serialized model; it carries the
format version. */
+ static final String MAGIC = "ONLP-FFDP-1";
+
+ /** The maximum number of (template position, embedding row) pairs the
scoring cache holds. */
+ static final int CACHE_ENTRY_LIMIT = 32_768;
+
+ /** The vocabulary label used in load errors for the word map. */
+ private static final String WORD_VOCABULARY = "word vocabulary";
+
+ /** The vocabulary label used in load errors for the tag map. */
+ private static final String TAG_VOCABULARY = "tag vocabulary";
+
+ /** The vocabulary label used in load errors for the label map. */
+ private static final String LABEL_VOCABULARY = "label vocabulary";
+
+ /** Maximum combined entries across the word, tag, and label maps. */
+ private static final int MAX_VOCABULARY_ENTRIES = 2_000_000;
+
+ /** Maximum transitions accepted from a serialized model. */
+ private static final int MAX_TRANSITIONS = 100_000;
+
+ /** Maximum embedding width accepted from a serialized model. */
+ static final int MAX_EMBEDDING_SIZE = 4_096;
+
+ /** Maximum hidden-layer width accepted from a serialized model. */
+ static final int MAX_HIDDEN_SIZE = 65_536;
+
+ /** Maximum float values allocated while loading a serialized model. */
+ static final long MAX_MODEL_FLOAT_VALUES = 100_000_000L;
+
+ /** U+03A3, GREEK CAPITAL LETTER SIGMA, the one code point with a contextual
lowering. */
+ private static final int GREEK_CAPITAL_SIGMA = 0x03A3;
+
+ /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the
capital. */
+ private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2';
+
+ /** The vocabulary key of every word, tag, or label the model has no
embedding row for. */
+ static final String UNKNOWN = "*UNK*";
+
+ /** The lazy scoring cache; {@code null} until {@link
#enableScoringCache()}. */
+ private volatile ContributionCache cache;
+
+ private final Map<String, Integer> wordIds;
+ private final Map<String, Integer> tagIds;
+ private final Map<String, Integer> labelIds;
+ private final String[] transitions;
+
+ private final int embeddingSize;
+ private final float[][] embeddings;
+ private final float[][] hiddenWeights;
+ private final float[] hiddenBias;
+ private final float[][] outputWeights;
+ private final float[] outputBias;
+
+ /**
+ * Assembles a model from its vocabularies and weights, copying the maps so
later
+ * changes by the caller do not reach the model.
+ *
+ * @param wordIds The word vocabulary, mapping each normalized word to its
row.
+ * @param tagIds The tag vocabulary.
+ * @param labelIds The dependency label vocabulary.
+ * @param transitions The transition inventory, indexed by output row.
+ * @param embeddingSize The embedding dimensionality.
+ * @param embeddings The embedding rows for words, tags, and labels.
+ * @param hiddenWeights The hidden layer weights.
+ * @param hiddenBias The hidden layer bias.
+ * @param outputWeights The output layer weights.
+ * @param outputBias The output layer bias.
+ */
+ FeedforwardDependencyModel(Map<String, Integer> wordIds, Map<String,
Integer> tagIds,
+ Map<String, Integer> labelIds, String[] transitions, int embeddingSize,
+ float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias,
+ float[][] outputWeights, float[] outputBias) {
+ this.wordIds = Map.copyOf(wordIds);
+ this.tagIds = Map.copyOf(tagIds);
+ this.labelIds = Map.copyOf(labelIds);
+ this.transitions = transitions;
+ this.embeddingSize = embeddingSize;
+ this.embeddings = embeddings;
+ this.hiddenWeights = hiddenWeights;
+ this.hiddenBias = hiddenBias;
+ this.outputWeights = outputWeights;
+ this.outputBias = outputBias;
+ }
+
+ /**
+ * Scores every transition for a configuration described by embedding row
indices.
+ *
+ * @param features The embedding rows of the configuration, as produced by
+ * {@link #featureIds(String[])}. Must not be {@code null}.
+ * @return One unnormalized score per transition, indexed like
+ * {@link #transitions()}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code features} is {@code
null}, does not
+ * have the required length, or contains an invalid embedding index.
+ * @throws IllegalStateException Thrown if the model produces a non-finite
transition score.
+ */
+ double[] score(int[] features) {
+ ParamChecks.requireNonNullArg(features, "features");
+ if (features.length != FeedforwardContext.FEATURE_COUNT) {
+ throw new IllegalArgumentException("features must contain "
+ + FeedforwardContext.FEATURE_COUNT + " embedding indices");
+ }
+ for (int i = 0; i < features.length; i++) {
+ if (features[i] < 0 || features[i] >= embeddings.length) {
+ throw new IllegalArgumentException(
+ "feature embedding out of range at index " + i + ": " +
features[i]);
+ }
+ }
+ final int hidden = hiddenBias.length;
+ final double[] h = new double[hidden];
+ for (int j = 0; j < hidden; j++) {
+ h[j] = hiddenBias[j];
+ }
+ final ContributionCache cache = this.cache;
+ for (int f = 0; f < features.length; f++) {
+ final int row = features[f];
+ final double[] contribution = cache == null ? null :
cache.contribution(this, f, row);
+ if (contribution != null) {
+ for (int j = 0; j < hidden; j++) {
+ h[j] += contribution[j];
+ }
+ } else {
+ final float[] embedding = embeddings[row];
+ final int offset = f * embeddingSize;
+ for (int j = 0; j < hidden; j++) {
+ final float[] weights = hiddenWeights[j];
+ double sum = 0.0;
+ for (int d = 0; d < embeddingSize; d++) {
+ sum += (double) weights[offset + d] * embedding[d];
+ }
+ h[j] += sum;
+ }
+ }
+ }
+ for (int j = 0; j < hidden; j++) {
+ h[j] = h[j] * h[j] * h[j];
+ }
+ final double[] scores = new double[transitions.length];
+ for (int o = 0; o < scores.length; o++) {
+ final float[] row = outputWeights[o];
+ double sum = outputBias[o];
+ for (int j = 0; j < hidden; j++) {
+ sum += row[j] * h[j];
+ }
+ if (!Double.isFinite(sum)) {
+ throw new IllegalStateException("the model produced a non-finite
transition score");
+ }
+ scores[o] = sum;
+ }
+ return scores;
+ }
+
+ /**
+ * Caches hidden-layer contributions by template position and embedding row.
+ *
+ * <p>Contributions retain double precision so cached and direct scoring use
the
+ * same values. Training and refinement use uncached copies; {@link #copy()}
+ * does not copy the cache.</p>
+ */
+ synchronized void enableScoringCache() {
+ if (cache == null) {
+ cache = new ContributionCache(FeedforwardContext.FEATURE_COUNT,
embeddings.length);
+ }
+ }
+
+ /**
+ * Stores contributions on first use up to a shared entry limit. Reference
tables
+ * are allocated for every template position and embedding row. Pairs beyond
the
+ * entry limit use direct scoring.
+ */
+ private static final class ContributionCache {
+
+ /** The cached contribution vectors, indexed by template position and
embedding row. */
+ private final AtomicReferenceArray<double[]>[] byPosition;
+
+ /** The number of pairs the cache may still add before the budget is
spent. */
+ private final AtomicInteger remaining = new
AtomicInteger(CACHE_ENTRY_LIMIT);
+
+ /**
+ * Creates an empty cache.
+ *
+ * @param positions The number of feature positions.
+ * @param rows The number of embedding rows a position can hold.
+ */
+ @SuppressWarnings("unchecked")
+ private ContributionCache(int positions, int rows) {
+ byPosition = new AtomicReferenceArray[positions];
+ for (int f = 0; f < positions; f++) {
+ byPosition[f] = new AtomicReferenceArray<>(rows);
+ }
+ }
+
+ /**
+ * Returns the cached hidden-layer contribution of one pair, computing and
+ * publishing it on first sight while the budget lasts.
+ *
+ * @param model The immutable model the contributions derive from.
+ * @param position The template position.
+ * @param row The embedding row at that position.
+ * @return The contribution vector, or {@code null} when the budget is
spent and
+ * the pair is not cached.
+ */
+ private double[] contribution(FeedforwardDependencyModel model, int
position, int row) {
+ final AtomicReferenceArray<double[]> slots = byPosition[position];
+ double[] contribution = slots.get(row);
+ if (contribution != null) {
+ return contribution;
+ }
+ if (!reserve()) {
+ return null;
+ }
+ boolean published = false;
+ try {
+ final int hidden = model.hiddenBias.length;
+ final float[] embedding = model.embeddings[row];
+ final int offset = position * model.embeddingSize;
+ contribution = new double[hidden];
+ for (int j = 0; j < hidden; j++) {
+ final float[] weights = model.hiddenWeights[j];
+ double sum = 0.0;
+ for (int d = 0; d < model.embeddingSize; d++) {
+ sum += (double) weights[offset + d] * embedding[d];
+ }
+ contribution[j] = sum;
+ }
+ published = slots.compareAndSet(row, null, contribution);
+ return published ? contribution : slots.get(row);
+ } finally {
+ if (!published) {
+ remaining.incrementAndGet();
+ }
+ }
+ }
+
+ /**
+ * Reserves capacity before allocating a contribution array.
+ *
+ * @return Whether one entry was reserved.
+ */
+ private boolean reserve() {
+ int available = remaining.get();
+ while (available > 0) {
+ if (remaining.compareAndSet(available, available - 1)) {
+ return true;
+ }
+ available = remaining.get();
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Maps the symbolic features of {@link FeedforwardContext} onto embedding
rows.
+ *
+ * @param symbols The symbolic features. Must not be {@code null}.
+ * @return The embedding row per feature. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code symbols} is {@code
null} or does
+ * not have the required length.
+ */
+ int[] featureIds(String[] symbols) {
Review Comment:
The lookup falls back to the unknown row, which construction and loading
guarantee, so the check and its exception are removed.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java:
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+/**
+ * The feature template of the feedforward parser: a fixed set of configuration
+ * positions whose words, tags, and arc labels are embedded and concatenated
into the
+ * network input, following
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>Positions: the top three stack and buffer items; the leftmost and
rightmost
+ * dependents of the top two stack items; and the leftmost dependent of the
leftmost
+ * dependent and rightmost dependent of the rightmost dependent of the top two
stack
+ * items, capturing second-order structure. Words and tags are read for all
positions,
+ * labels only for the dependent positions, whose relations are already
assigned.</p>
+ */
+final class FeedforwardContext {
+
+ /** The number of positions whose word and tag are embedded. */
+ static final int POSITIONS = 14;
+
+ /** The number of dependent positions whose arc label is embedded. */
+ static final int LABEL_POSITIONS = 8;
+
+ /** The number of symbolic features in one parser configuration. */
+ static final int FEATURE_COUNT = 2 * POSITIONS + LABEL_POSITIONS;
+
+ /** The index of the first dependent position; the stack and buffer items
precede it. */
+ private static final int FIRST_DEPENDENT_POSITION = 6;
+
+ /** Prevents construction of this utility class. */
+ private FeedforwardContext() {
+ }
+
+ /**
+ * Extracts the symbolic features of a configuration: {@link #POSITIONS}
words, then
+ * {@link #POSITIONS} tags, then {@link #LABEL_POSITIONS} labels; absent
positions
+ * yield {@code null} entries, which the vocabulary maps to its padding
symbol.
+ *
+ * @param state The configuration to describe. Must not be {@code null}.
+ * @param tokens The sentence tokens. Must not be {@code null}.
+ * @param tags The part-of-speech tags aligned with {@code tokens}. Must not
be
+ * {@code null}.
+ * @return The symbolic features, in the order described above. Never {@code
null}.
+ */
+ static String[] extract(ArcStandardState state, String[] tokens, String[]
tags) {
+ final int s0 = state.stack(0);
+ final int s1 = state.stack(1);
+ final int[] positions = {
+ s0, s1, state.stack(2),
+ state.buffer(0), state.buffer(1), state.buffer(2),
+ leftmost(state, s0), rightmost(state, s0),
+ leftmost(state, s1), rightmost(state, s1),
+ leftmost(state, leftmost(state, s0)), rightmost(state,
rightmost(state, s0)),
+ leftmost(state, leftmost(state, s1)), rightmost(state,
rightmost(state, s1))
+ };
+ final String[] features = new String[FEATURE_COUNT];
+ for (int i = 0; i < POSITIONS; i++) {
+ features[i] = symbol(tokens, positions[i]);
+ features[POSITIONS + i] = symbol(tags, positions[i]);
+ }
+ for (int i = 0; i < LABEL_POSITIONS; i++) {
+ final int position = positions[FIRST_DEPENDENT_POSITION + i];
+ features[2 * POSITIONS + i] =
+ position >= 0 ? state.assignedRelation(position) : null;
+ }
+ return features;
+ }
+
+ /** The leftmost dependent of a position, or the absence marker for absent
positions. */
+ private static int leftmost(ArcStandardState state, int index) {
Review Comment:
Added `@param`/`@return`, `@since 3.0.0`, and a note on the per-sentence
scratch state.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java:
##########
@@ -0,0 +1,1176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Function;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ParamChecks;
+
+/**
+ * Trains the {@link FeedforwardDependencyModel}: oracle-derived transition
examples,
+ * minibatch AdaGrad over a softmax cross-entropy loss, cube activation, and
inverted
+ * dropout on the hidden layer, the training recipe of
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>Words below the frequency cutoff share a learned unknown embedding;
absent
+ * template positions share a learned padding embedding. Non-projective
samples have no
+ * arc-standard derivation and are skipped. Training is deterministic for a
fixed
+ * {@link Settings#seed()}.</p>
+ *
+ * @since 3.0.0
+ */
+public final class FeedforwardDependencyTrainer {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(FeedforwardDependencyTrainer.class);
+
+ /** The AdaGrad denominator offset that keeps the first steps finite. */
+ private static final double ADAGRAD_EPSILON = 1e-6;
+
+ /** The derivative factor of the cube activation, {@code d/dx x^3 = 3x^2}. */
+ private static final double CUBE_DERIVATIVE_FACTOR = 3.0;
+
+ /** The numerator of the Glorot uniform bound, {@code sqrt(6 / (fanIn +
fanOut))}. */
+ private static final double GLOROT_NUMERATOR = 6.0;
+
+ /** The half-width of the uniform range embeddings are drawn from. */
+ private static final double EMBEDDING_INIT_SCALE = 0.01;
+
+ /** The smallest probability the reported training loss takes the log of. */
+ private static final double LOSS_PROBABILITY_FLOOR = 1e-12;
+
+ /** The sentence loss reported when refinement made no update for a
sentence. */
+ private static final double NO_UPDATE = -1.0;
+
+ /** Special-symbol rows present before any training vocabulary is added. */
+ private static final int MIN_VOCABULARY_ROWS = 8;
+
+ /** SHIFT and at least one RIGHT_ARC are required to parse a sentence. */
+ private static final int MIN_TRANSITIONS = 2;
+
+ /** Prevents construction of this utility class. */
+ private FeedforwardDependencyTrainer() {
+ }
+
+ /**
+ * The training hyperparameters.
+ *
+ * @param embeddingSize The embedding dimensionality. Must be positive.
+ * @param hiddenSize The hidden layer width. Must be positive.
+ * @param epochs The number of passes over the examples. Must be positive.
+ * @param batchSize The minibatch size. Must be positive.
+ * @param learningRate The AdaGrad step size. Must be positive.
+ * @param l2 The L2 penalty applied to the dense weights. Must not be
negative.
+ * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}.
+ * @param wordCutoff The minimum frequency for a word to get its own
embedding. Must
+ * not be negative.
+ * @param seed The random seed making a run reproducible.
+ */
+ public record Settings(int embeddingSize, int hiddenSize, int epochs, int
batchSize,
+ double learningRate, double l2, double dropout, int wordCutoff, long
seed) {
+
+ /** The default embedding dimensionality. */
+ private static final int DEFAULT_EMBEDDING_SIZE = 50;
+
+ /** The default hidden layer width. */
+ private static final int DEFAULT_HIDDEN_SIZE = 200;
+
+ /** The default number of epochs. */
+ private static final int DEFAULT_EPOCHS = 10;
+
+ /** The default minibatch size. */
+ private static final int DEFAULT_BATCH_SIZE = 256;
+
+ /** The default AdaGrad step size. */
+ private static final double DEFAULT_LEARNING_RATE = 0.02;
+
+ /** The default L2 penalty. */
+ private static final double DEFAULT_L2 = 1e-8;
+
+ /** The default hidden dropout probability. */
+ private static final double DEFAULT_DROPOUT = 0.5;
+
+ /** The default minimum word frequency for an embedding row. */
+ private static final int DEFAULT_WORD_CUTOFF = 2;
+
+ /** The default random seed. */
+ static final long DEFAULT_SEED = 17L;
+
+ /**
+ * Validates the hyperparameters.
+ *
+ * @throws IllegalArgumentException Thrown if a value is outside its
documented
+ * range.
+ */
+ public Settings {
+ requirePositive(embeddingSize, "embeddingSize");
+ requirePositive(hiddenSize, "hiddenSize");
+ requirePositive(epochs, "epochs");
+ requirePositive(batchSize, "batchSize");
+ if (embeddingSize > FeedforwardDependencyModel.MAX_EMBEDDING_SIZE) {
+ throw new IllegalArgumentException("embeddingSize exceeds the model
format limit: "
+ + embeddingSize);
+ }
+ if (hiddenSize > FeedforwardDependencyModel.MAX_HIDDEN_SIZE) {
+ throw new IllegalArgumentException("hiddenSize exceeds the model
format limit: "
+ + hiddenSize);
+ }
+ final long minimumModelValues = modelFloatValues(MIN_VOCABULARY_ROWS,
+ MIN_TRANSITIONS, embeddingSize, hiddenSize);
+ if (minimumModelValues >
FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) {
+ throw new IllegalArgumentException("embeddingSize and hiddenSize
exceed the model "
+ + "format allocation limit");
+ }
+ if (ParamChecks.requireFinite(learningRate, "learningRate") <= 0.0) {
+ throw new IllegalArgumentException("learningRate must be positive: " +
learningRate);
+ }
+ ParamChecks.requireNonNegative(ParamChecks.requireFinite(l2, "l2"),
"l2");
+ if (!(dropout >= 0.0 && dropout < 1.0)) {
+ throw new IllegalArgumentException("dropout must be in [0, 1): " +
dropout);
+ }
+ ParamChecks.requireNonNegative(wordCutoff, "wordCutoff");
+ }
+
+ /**
+ * Rejects a hyperparameter that is not positive.
+ *
+ * @param value The value to check.
+ * @param name The hyperparameter name used in the exception message.
+ * @throws IllegalArgumentException Thrown if {@code value} is not
positive.
+ */
+ private static void requirePositive(int value, String name) {
+ if (value <= 0) {
+ throw new IllegalArgumentException(name + " must be positive: " +
value);
+ }
+ }
+
+ /**
+ * @return The default hyperparameters. Never {@code null}.
+ */
+ public static Settings defaults() {
+ return new Settings(DEFAULT_EMBEDDING_SIZE, DEFAULT_HIDDEN_SIZE,
DEFAULT_EPOCHS,
+ DEFAULT_BATCH_SIZE, DEFAULT_LEARNING_RATE, DEFAULT_L2,
DEFAULT_DROPOUT,
+ DEFAULT_WORD_CUTOFF, DEFAULT_SEED);
+ }
+ }
+
+ /**
+ * Trains a model from dependency samples.
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null} or
no
+ * trainable example can be derived from the samples.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings) throws IOException {
+ return train(samples, settings, null);
+ }
+
+ /**
+ * Trains a model from dependency samples, seeding word embeddings from a
pretrained
+ * source.
+ *
+ * <p>The provider is consulted once per vocabulary word during
initialization; words
Review Comment:
Trimmed to the contract.
> Add a feedforward neural dependency parser
> ------------------------------------------
>
> Key: OPENNLP-1991
> URL: https://issues.apache.org/jira/browse/OPENNLP-1991
> Project: OpenNLP
> Issue Type: Improvement
> Reporter: Kristian Rickert
> Assignee: Kristian Rickert
> Priority: Major
> Time Spent: 10m
> Remaining Estimate: 0h
>
> Add a feedforward neural implementation of the dependency parser API
> introduced by OPENNLP-547. Use OpenNLP’s standard model container, artifact
> serializers, TrainingParameters, factory, and CLI conventions. Include
> deterministic training, greedy and beam decoding, model round-trip tests, and
> held-out Universal Dependencies evaluation. Address the deferred review
> feedback on shared scoring/training logic and parser-owned caches. Implement
> this without new dependencies.
> The list above may still get this ticket to further split. For now this is a
> placeholder to capture the code that was moved out of OPENNLP-547. I'll edit
> after 547 is merged.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
