This is an automated email from the ASF dual-hosted git repository. krickert pushed a commit to branch feat/GH-1469-feedforward-parser-cli in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit f21d77b63c1e924c67fa97d103dd7c80f4facb88 Author: Kristian Rickert <[email protected]> AuthorDate: Sun Oct 11 05:56:28 2026 -0400 GH-1469: Add command-line tools for the feedforward dependency parser FeedforwardDependencyParserTrainer trains a FeedforwardDependencyModel from the depparse sample formats. The -params file takes the trainer keys, and without a file the trainer defaults apply. -refineParams refines the model with beam search; with -baseModel the tool refines a saved model instead of training one. FeedforwardDependencyParser parses tagged lines from standard input and FeedforwardDependencyParserEvaluator logs UAS and LAS in the format of the maxent tools. Both take -beamSize, defaulting to the beam size a refined model records in the manifest, or 1. The manual's dependency chapter documents the tools. ModelLoader now reports a model of another component as "Model has invalid format" for all loaders instead of a raw IllegalArgumentException. The CoNLL-U output loop and the scoring step are shared with the maxent tools, and the depparse CLI tests use one fixture class. Signed-off-by: Kristian Rickert <[email protected]> --- .../src/main/java/opennlp/tools/cmdline/CLI.java | 6 + .../java/opennlp/tools/cmdline/ModelLoader.java | 9 + .../depparse/DependencyParserEvaluatorTool.java | 33 +- .../cmdline/depparse/DependencyParserMETool.java | 13 +- .../depparse/FeedforwardDependencyModelLoader.java | 46 ++ .../FeedforwardDependencyParserEvaluatorTool.java | 77 +++ .../depparse/FeedforwardDependencyParserTool.java | 133 ++++++ .../FeedforwardDependencyParserTrainerTool.java | 133 ++++++ .../depparse/DependencyParserToolsTest.java | 91 +--- .../depparse/DependencyToolTestSupport.java | 103 ++++ .../FeedforwardDependencyParserToolsTest.java | 526 +++++++++++++++++++++ opennlp-docs/src/docbkx/dependency.xml | 27 ++ 12 files changed, 1116 insertions(+), 81 deletions(-) diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/CLI.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/CLI.java index ff682d053..dbb259b1c 100644 --- a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/CLI.java +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/CLI.java @@ -37,6 +37,9 @@ import opennlp.tools.cmdline.depparse.DependencyParserCrossValidatorTool; import opennlp.tools.cmdline.depparse.DependencyParserEvaluatorTool; import opennlp.tools.cmdline.depparse.DependencyParserMETool; import opennlp.tools.cmdline.depparse.DependencyParserTrainerTool; +import opennlp.tools.cmdline.depparse.FeedforwardDependencyParserEvaluatorTool; +import opennlp.tools.cmdline.depparse.FeedforwardDependencyParserTool; +import opennlp.tools.cmdline.depparse.FeedforwardDependencyParserTrainerTool; import opennlp.tools.cmdline.dictionary.DictionaryBuilderTool; import opennlp.tools.cmdline.doccat.DoccatConverterTool; import opennlp.tools.cmdline.doccat.DoccatCrossValidatorTool; @@ -171,6 +174,9 @@ public final class CLI { tools.add(new DependencyParserEvaluatorTool()); tools.add(new DependencyParserCrossValidatorTool()); tools.add(new DependencyParserMETool()); + tools.add(new FeedforwardDependencyParserTrainerTool()); + tools.add(new FeedforwardDependencyParserEvaluatorTool()); + tools.add(new FeedforwardDependencyParserTool()); // Chunker tools.add(new ChunkerMETool()); diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/ModelLoader.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/ModelLoader.java index 180aade29..6a1d10052 100644 --- a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/ModelLoader.java +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/ModelLoader.java @@ -65,6 +65,15 @@ public abstract class ModelLoader<T> { catch (InvalidFormatException e) { throw new TerminateToolException(-1, "Model has invalid format", e); } + catch (IllegalArgumentException e) { + // BaseModel reports a model of another component, whose manifest names a tool + // factory this model cannot use, as an IllegalArgumentException with the + // InvalidFormatException as its cause. + if (e.getCause() instanceof InvalidFormatException) { + throw new TerminateToolException(-1, "Model has invalid format", e.getCause()); + } + throw e; + } catch (IOException e) { throw new TerminateToolException(-1, "IO error while loading model file '" + modelFile + "'", e); } diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserEvaluatorTool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserEvaluatorTool.java index 8d25e52a2..8e86dcad5 100644 --- a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserEvaluatorTool.java +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserEvaluatorTool.java @@ -18,6 +18,7 @@ package opennlp.tools.cmdline.depparse; import java.io.IOException; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +27,7 @@ import opennlp.tools.cmdline.AbstractEvaluatorTool; import opennlp.tools.cmdline.params.EvaluatorParams; import opennlp.tools.depparse.DependencyEvaluationMonitor; import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyParser; import opennlp.tools.depparse.DependencyParserME; import opennlp.tools.depparse.DependencySample; import opennlp.tools.util.ObjectStream; @@ -51,21 +53,40 @@ public class DependencyParserEvaluatorTool @Override public void run(String format, String[] args) { super.run(format, args); - try (ObjectStream<DependencySample> samples = sampleStream) { + try { + evaluate(() -> new DependencyParserME(new DependencyModelLoader().load(params.getModel())), + sampleStream, params.getMisclassified(), logger); + } catch (IOException e) { + throw createTerminationIOException(e); + } + } + + /** + * Scores a parser on samples and logs the attachment scores over all tokens and over + * the tokens that are not punctuation. Closes the samples, also when the parser + * cannot be created. + * + * @param parser Creates the parser to score, typically by loading its model. Must not + * be {@code null}. + * @param samples The gold samples. Must not be {@code null}. + * @param misclassified Whether to print each misparsed sentence next to its gold tree. + * @param logger The logger to report the scores to. Must not be {@code null}. + * @throws IOException Thrown if reading the samples fails. + */ + static void evaluate(Supplier<DependencyParser> parser, ObjectStream<DependencySample> samples, + boolean misclassified, Logger logger) throws IOException { + try (samples) { DependencyEvaluationMonitor errorListener = null; - if (params.getMisclassified()) { + if (misclassified) { errorListener = new DependencyEvaluationErrorListener(); } - DependencyEvaluator evaluator = new DependencyEvaluator(new DependencyParserME( - new DependencyModelLoader().load(params.getModel())), errorListener); + DependencyEvaluator evaluator = new DependencyEvaluator(parser.get(), errorListener); evaluator.evaluate(samples); logger.info("Tokens: {}; UAS: {}; LAS: {}", evaluator.getWordCount(), evaluator.getUas(), evaluator.getLas()); logger.info("Tokens excluding punctuation: {}; UAS: {}; LAS: {}", evaluator.getWordCountExcludingPunctuation(), evaluator.getUasExcludingPunctuation(), evaluator.getLasExcludingPunctuation()); - } catch (IOException e) { - throw createTerminationIOException(e); } } } diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserMETool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserMETool.java index 66cc5342a..ae716ae12 100644 --- a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserMETool.java +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserMETool.java @@ -26,6 +26,7 @@ import opennlp.tools.cmdline.CLI; import opennlp.tools.cmdline.CmdLineUtil; import opennlp.tools.cmdline.SystemInputStreamFactory; import opennlp.tools.cmdline.TerminateToolException; +import opennlp.tools.depparse.DependencyParser; import opennlp.tools.depparse.DependencyParserME; import opennlp.tools.depparse.DependencyTree; import opennlp.tools.postag.POSSample; @@ -60,7 +61,17 @@ public class DependencyParserMETool extends BasicCmdLineTool { if (args.length != 1) { throw new TerminateToolException(1, getHelp()); } - DependencyParserME parser = new DependencyParserME(new DependencyModelLoader().load(new File(args[0]))); + parseTaggedSentences(new DependencyParserME(new DependencyModelLoader().load(new File(args[0])))); + } + + /** + * Parses the POS-tagged sentences on standard input, one per line in the + * {@code token_TAG} format, and prints each tree in basic CoNLL-U followed by a blank + * line. Blank input lines are skipped. + * + * @param parser The parser to apply. Must not be {@code null}. + */ + static void parseTaggedSentences(DependencyParser parser) { PrintWriter out = new PrintWriter(System.out, true); try (ObjectStream<String> lines = new PlainTextByLineStream( new SystemInputStreamFactory(), SystemInputStreamFactory.encoding())) { diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyModelLoader.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyModelLoader.java new file mode 100644 index 000000000..099b2948d --- /dev/null +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyModelLoader.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.depparse; + +import java.io.IOException; +import java.io.InputStream; + +import opennlp.tools.cmdline.ModelLoader; +import opennlp.tools.commons.Internal; +import opennlp.tools.depparse.FeedforwardDependencyModel; + +/** + * Loads a {@link FeedforwardDependencyModel} for the command line tools. + * <p> + * <b>Note:</b> Do not use this class, internal use only! + */ +@Internal +public class FeedforwardDependencyModelLoader extends ModelLoader<FeedforwardDependencyModel> { + + /** Initializes a loader that names the model "Feedforward dependency parser" in its messages. */ + public FeedforwardDependencyModelLoader() { + super("Feedforward dependency parser"); + } + + /** {@inheritDoc} */ + @Override + protected FeedforwardDependencyModel loadModel(InputStream modelIn) throws IOException { + return new FeedforwardDependencyModel(modelIn); + } + +} diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserEvaluatorTool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserEvaluatorTool.java new file mode 100644 index 000000000..381bf2a63 --- /dev/null +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserEvaluatorTool.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.depparse; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.cmdline.AbstractEvaluatorTool; +import opennlp.tools.cmdline.ArgumentParser.OptionalParameter; +import opennlp.tools.cmdline.ArgumentParser.ParameterDescription; +import opennlp.tools.cmdline.depparse.FeedforwardDependencyParserEvaluatorTool.EvalToolParams; +import opennlp.tools.cmdline.params.EvaluatorParams; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.depparse.FeedforwardDependencyParser; + +/** + * Reports labeled and unlabeled attachment scores of a {@link FeedforwardDependencyParser} + * against gold trees. + */ +public class FeedforwardDependencyParserEvaluatorTool + extends AbstractEvaluatorTool<DependencySample, EvalToolParams> { + + /** The evaluator parameters plus the beam size to parse with. */ + interface EvalToolParams extends EvaluatorParams { + + @ParameterDescription(valueName = "n", description = "the number of transition sequences " + + "to keep per step. Defaults to the beam size the model was refined with, or 1 " + + "(greedy) for a model that was not refined.") + @OptionalParameter + Integer getBeamSize(); + } + + private static final Logger logger = + LoggerFactory.getLogger(FeedforwardDependencyParserEvaluatorTool.class); + + /** Creates the evaluator tool. */ + public FeedforwardDependencyParserEvaluatorTool() { + super(DependencySample.class, EvalToolParams.class); + } + + /** {@inheritDoc} */ + @Override + public String getShortDescription() { + return "Measures feedforward dependency parser attachment scores"; + } + + /** {@inheritDoc} */ + @Override + public void run(String format, String[] args) { + super.run(format, args); + try { + DependencyParserEvaluatorTool.evaluate( + () -> FeedforwardDependencyParserTool.createParser( + new FeedforwardDependencyModelLoader().load(params.getModel()), params.getBeamSize()), + sampleStream, params.getMisclassified(), logger); + } catch (IOException e) { + throw createTerminationIOException(e); + } + } +} diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTool.java new file mode 100644 index 000000000..1bb21c8d4 --- /dev/null +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTool.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.depparse; + +import java.io.File; + +import opennlp.tools.cmdline.BasicCmdLineTool; +import opennlp.tools.cmdline.CLI; +import opennlp.tools.cmdline.TerminateToolException; +import opennlp.tools.depparse.FeedforwardDependencyModel; +import opennlp.tools.depparse.FeedforwardDependencyParser; +import opennlp.tools.ml.BeamSearch; + +/** + * Parses POS-tagged sentences from standard input with a {@link FeedforwardDependencyParser} + * and prints basic CoNLL-U trees. + */ +public class FeedforwardDependencyParserTool extends BasicCmdLineTool { + + /** The option that sets the beam size. */ + static final String BEAM_SIZE_OPTION = "-beamSize"; + + /** The beam size at which decoding is greedy. */ + private static final int GREEDY_BEAM_SIZE = 1; + + /** {@inheritDoc} */ + @Override + public String getShortDescription() { + return "Parses dependency trees from POS-tagged sentences with the feedforward parser"; + } + + /** {@inheritDoc} */ + @Override + public String getHelp() { + return "Usage: " + CLI.CMD + " " + getName() + " [" + BEAM_SIZE_OPTION + " n] model" + + " < tagged-sentences\n" + + BEAM_SIZE_OPTION + " n: keep n transition sequences per step. Defaults to the beam" + + " size the model was refined with, or 1 (greedy) for a model that was not refined."; + } + + /** {@inheritDoc} */ + @Override + public void run(String[] args) { + final Integer beamSize; + final String modelFile; + if (args.length == 1) { + beamSize = null; + modelFile = args[0]; + } else if (args.length == 3 && BEAM_SIZE_OPTION.equals(args[0])) { + beamSize = parseBeamSize(args[1]); + modelFile = args[2]; + } else { + throw new TerminateToolException(1, getHelp()); + } + DependencyParserMETool.parseTaggedSentences( + createParser(new FeedforwardDependencyModelLoader().load(new File(modelFile)), beamSize)); + } + + /** + * @param value The {@value #BEAM_SIZE_OPTION} argument. + * @return The beam size. Not checked for range. + * @throws TerminateToolException Thrown if {@code value} is not a decimal integer. + */ + private static int parseBeamSize(String value) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new TerminateToolException(1, BEAM_SIZE_OPTION + " must be an integer: " + value, e); + } + } + + /** + * Creates a parser with the requested beam size, or the beam size of the model when + * none is requested. + * + * @param model The model to parse with. Must not be {@code null}. + * @param requested The beam size given on the command line, or {@code null}. + * @return The parser. Never {@code null}. + * @throws TerminateToolException Thrown if the beam size is not positive or the model + * records an invalid beam size. + */ + static FeedforwardDependencyParser createParser(FeedforwardDependencyModel model, + Integer requested) { + try { + return new FeedforwardDependencyParser(model, beamSize(model, requested)); + } catch (IllegalArgumentException e) { + throw new TerminateToolException(-1, e.getMessage(), e); + } + } + + /** + * Resolves the beam size to parse with. A model refined with beam search records its + * beam size in the manifest under {@value BeamSearch#BEAM_SIZE_PARAMETER}; a locally + * trained model records none and is parsed greedily. + * + * @param model The model to parse with. Must not be {@code null}. + * @param requested The beam size given on the command line, or {@code null}. + * @return {@code requested} if not {@code null}, else the beam size of the model, else + * {@code 1}. Not checked for range. + * @throws TerminateToolException Thrown if the model records a beam size that is not + * a decimal integer. + */ + static int beamSize(FeedforwardDependencyModel model, Integer requested) { + if (requested != null) { + return requested; + } + final String recorded = model.getManifestProperty(BeamSearch.BEAM_SIZE_PARAMETER); + if (recorded == null) { + return GREEDY_BEAM_SIZE; + } + try { + return Integer.parseInt(recorded); + } catch (NumberFormatException e) { + throw new TerminateToolException(-1, "The model records an invalid " + + BeamSearch.BEAM_SIZE_PARAMETER + ": " + recorded, e); + } + } +} diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTrainerTool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTrainerTool.java new file mode 100644 index 000000000..d2f8f2476 --- /dev/null +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserTrainerTool.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.depparse; + +import java.io.File; +import java.io.IOException; + +import opennlp.tools.cmdline.AbstractTrainerTool; +import opennlp.tools.cmdline.ArgumentParser.OptionalParameter; +import opennlp.tools.cmdline.ArgumentParser.ParameterDescription; +import opennlp.tools.cmdline.CmdLineUtil; +import opennlp.tools.cmdline.TerminateToolException; +import opennlp.tools.cmdline.depparse.FeedforwardDependencyParserTrainerTool.TrainerParams; +import opennlp.tools.cmdline.params.TrainingToolParams; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.depparse.FeedforwardDependencyModel; +import opennlp.tools.depparse.FeedforwardDependencyTrainer; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.TrainingParameters; + +/** + * Trains and saves a {@link FeedforwardDependencyModel}, optionally refining it with beam + * search, or refines an existing model. + * + * <p>The {@code -params} and {@code -refineParams} files hold the keys that + * {@link FeedforwardDependencyTrainer} reads; absent keys take the trainer's defaults.</p> + */ +public class FeedforwardDependencyParserTrainerTool + extends AbstractTrainerTool<DependencySample, TrainerParams> { + + /** The training parameters plus the optional refinement step. */ + interface TrainerParams extends TrainingToolParams { + + @ParameterDescription(valueName = "paramsFile", description = "refines the model with beam " + + "search, reading the refinement parameters from this file.") + @OptionalParameter + String getRefineParams(); + + @ParameterDescription(valueName = "modelFile", description = "refines this model instead " + + "of training a new one. Requires " + REFINE_PARAMS_OPTION + ".") + @OptionalParameter + File getBaseModel(); + } + + /** The option that names the refinement parameters file. */ + private static final String REFINE_PARAMS_OPTION = "-refineParams"; + + /** The option that names the model to refine. */ + private static final String BASE_MODEL_OPTION = "-baseModel"; + + /** Creates the trainer tool. */ + public FeedforwardDependencyParserTrainerTool() { + super(DependencySample.class, TrainerParams.class); + } + + /** {@inheritDoc} */ + @Override + public String getShortDescription() { + return "Trains a feedforward dependency parser from CoNLL-U samples"; + } + + /** {@inheritDoc} */ + @Override + public void run(String format, String[] args) { + super.run(format, args); + try (ObjectStream<DependencySample> samples = sampleStream) { + if (params.getBaseModel() != null) { + if (params.getRefineParams() == null) { + throw new TerminateToolException(1, + BASE_MODEL_OPTION + " requires " + REFINE_PARAMS_OPTION); + } + if (params.getParams() != null) { + throw new TerminateToolException(1, "-params does not apply with " + BASE_MODEL_OPTION + + "; refinement reads " + REFINE_PARAMS_OPTION); + } + } + CmdLineUtil.checkOutputFile("feedforward dependency parser model", params.getModel()); + final TrainingParameters refineParams = params.getRefineParams() == null + ? null : CmdLineUtil.loadTrainingParameters(params.getRefineParams(), false); + + FeedforwardDependencyModel model; + if (params.getBaseModel() != null) { + model = new FeedforwardDependencyModelLoader().load(params.getBaseModel()); + if (!params.getLang().equals(model.getLanguage())) { + throw new TerminateToolException(1, "-lang " + params.getLang() + + " does not match the base model language " + model.getLanguage()); + } + } else { + mlParams = loadTrainingParameters(params.getParams()); + model = FeedforwardDependencyTrainer.train(params.getLang(), samples, mlParams); + if (refineParams != null) { + samples.reset(); + } + } + if (refineParams != null) { + model = FeedforwardDependencyTrainer.refine(model, samples, refineParams); + } + CmdLineUtil.writeModel("feedforward dependency parser", params.getModel(), model); + } catch (IOException e) { + throw createTerminationIOException(e); + } catch (IllegalArgumentException | IllegalStateException e) { + throw new TerminateToolException(-1, e.getMessage(), e); + } + } + + /** + * Loads the training parameters file named on the command line. Without a file the + * parameters are empty, so every key takes the default of + * {@link FeedforwardDependencyTrainer}. + * + * @param paramsFile The parameters file name, or {@code null} if none was given. + * @return The training parameters. Never {@code null}. + */ + private static TrainingParameters loadTrainingParameters(String paramsFile) { + final TrainingParameters loaded = CmdLineUtil.loadTrainingParameters(paramsFile, false); + return loaded == null ? new TrainingParameters() : loaded; + } +} diff --git a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyParserToolsTest.java b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyParserToolsTest.java index 9ff205367..fc233e097 100644 --- a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyParserToolsTest.java +++ b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyParserToolsTest.java @@ -28,16 +28,11 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.slf4j.LoggerFactory; import opennlp.tools.cmdline.CLI; import opennlp.tools.cmdline.StreamFactoryRegistry; @@ -48,6 +43,17 @@ import opennlp.tools.depparse.DependencySample; import opennlp.tools.depparse.DependencyTree; import opennlp.tools.util.ObjectStream; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.EXPECTED_REVERSED; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.GOLD; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PARSED_SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PERFECT_SCORES; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PERFECT_SCORES_EXCLUDING_PUNCTUATION; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PREDICTED_SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.REPETITIONS; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.REVERSED_SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.UTF_8_REQUIRED; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.logOf; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -55,42 +61,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** Exercises CLI registration, CoNLL-U input, training, persistence and inference. */ public class DependencyParserToolsTest { - /** A two-token sentence in CoNLL-U; the tests repeat it to build a corpus. */ - private static final String SENTENCE = """ - 1\tdogs\tdog\tNOUN\tNNS\t_\t2\tnsubj\t_\t_ - 2\trun\trun\tVERB\tVBP\t_\t0\troot\t_\t_ - - """; - - /** How often {@link #SENTENCE} is repeated in the training data. */ - private static final int REPETITIONS = 20; - - /** The gold tree of {@link #SENTENCE}. */ - private static final DependencyTree GOLD = - DependencyTree.of(new int[] {1, -1}, new String[] {"nsubj", "root"}); - - /** The sentence with the heads swapped, which a parser trained on {@link #SENTENCE} misparses. */ - private static final String REVERSED_SENTENCE = """ - 1\tdogs\tdog\tNOUN\tNNS\t_\t0\troot\t_\t_ - 2\trun\trun\tVERB\tVBP\t_\t1\tobj\t_\t_ - - """; - - /** The gold tree of {@link #REVERSED_SENTENCE}. */ + /** The gold tree of {@link DependencyToolTestSupport#REVERSED_SENTENCE}. */ private static final DependencyTree REVERSED = DependencyTree.of(new int[] {-1, 0}, new String[] {"root", "obj"}); - /** The expected block the error listener prints for {@link #REVERSED_SENTENCE}. */ - private static final String EXPECTED_REVERSED = "Expected: {\n" - + "1\tdogs\tNOUN\t0\troot" + System.lineSeparator() - + "2\trun\tVERB\t1\tobj" + System.lineSeparator() + "}"; - - /** The predicted block the error listener prints for the parse of {@link #SENTENCE}. */ - private static final String PREDICTED_SENTENCE = "Predicted: {\n" - + "1\tdogs\tNOUN\t2\tnsubj" + System.lineSeparator() - + "2\trun\tVERB\t0\troot" + System.lineSeparator() + "}"; - - /** The sentence with the subject relabeled, so only a label differs from {@link #SENTENCE}. */ + /** + * The sentence with the subject relabeled, so only a label differs from + * {@link DependencyToolTestSupport#SENTENCE}. + */ private static final String RELABELED_SENTENCE = """ 1\tdogs\tdog\tNOUN\tNNS\t_\t2\tnmod\t_\t_ 2\trun\trun\tVERB\tVBP\t_\t0\troot\t_\t_ @@ -105,16 +83,6 @@ public class DependencyParserToolsTest { /** The message prefix of a tool ended by an unusable argument value. */ private static final String INVALID_ARGUMENT = "Invalid argument: "; - /** The message of the CoNLL-U format for an encoding other than UTF-8. */ - private static final String UTF_8_REQUIRED = "CoNLL-U data must use UTF-8"; - - /** The scores logged for a parser that reproduces all 40 tokens of the data. */ - private static final String PERFECT_SCORES = "Tokens: 40; UAS: 1.0; LAS: 1.0"; - - /** The punctuation-free scores logged for the same parser; the data has no punctuation. */ - private static final String PERFECT_SCORES_EXCLUDING_PUNCTUATION = - "Tokens excluding punctuation: 40; UAS: 1.0; LAS: 1.0"; - @TempDir private Path dir; @@ -134,29 +102,6 @@ public class DependencyParserToolsTest { "-lang", "eng", "-data", data.toString(), "-model", model.toString()}); } - /** - * Runs a tool while capturing the INFO messages its logger emits. - * - * @param tool The tool class whose logger is captured. - * @param run Runs the tool. - * @return The formatted log messages, in order. Never {@code null}. - */ - private static List<String> logOf(Class<?> tool, Runnable run) { - final Logger logger = (Logger) LoggerFactory.getLogger(tool); - final Level previous = logger.getLevel(); - final ListAppender<ILoggingEvent> appender = new ListAppender<>(); - appender.start(); - logger.addAppender(appender); - logger.setLevel(Level.INFO); - try { - run.run(); - } finally { - logger.setLevel(previous); - logger.detachAppender(appender); - } - return appender.list.stream().map(ILoggingEvent::getFormattedMessage).toList(); - } - @Test void testToolsAreRegistered() { assertTrue(CLI.getToolNames().contains("DependencyParserME")); @@ -203,9 +148,7 @@ public class DependencyParserToolsTest { System.setIn(previousIn); System.setOut(previousOut); } - assertEquals("1\tdogs\t_\t_\tNOUN\t_\t2\tnsubj\t_\t_\n" - + "2\trun\t_\t_\tVERB\t_\t0\troot\t_\t_\n\n", - output.toString(StandardCharsets.UTF_8).replace("\r\n", "\n")); + assertEquals(PARSED_SENTENCE, output.toString(StandardCharsets.UTF_8).replace("\r\n", "\n")); } @Test diff --git a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyToolTestSupport.java b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyToolTestSupport.java new file mode 100644 index 000000000..2bd625f6d --- /dev/null +++ b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyToolTestSupport.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.depparse; + +import java.util.List; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyTree; + +/** The CoNLL-U data, expected output, and log capture shared by the dependency parser tool tests. */ +final class DependencyToolTestSupport { + + private DependencyToolTestSupport() { + } + + /** A two-token sentence in CoNLL-U; the tests repeat it to build a corpus. */ + static final String SENTENCE = """ + 1\tdogs\tdog\tNOUN\tNNS\t_\t2\tnsubj\t_\t_ + 2\trun\trun\tVERB\tVBP\t_\t0\troot\t_\t_ + + """; + + /** How often {@link #SENTENCE} is repeated in the training data. */ + static final int REPETITIONS = 20; + + /** The gold tree of {@link #SENTENCE}. */ + static final DependencyTree GOLD = + DependencyTree.of(new int[] {1, -1}, new String[] {"nsubj", "root"}); + + /** The sentence with the heads swapped, which a parser trained on {@link #SENTENCE} misparses. */ + static final String REVERSED_SENTENCE = """ + 1\tdogs\tdog\tNOUN\tNNS\t_\t0\troot\t_\t_ + 2\trun\trun\tVERB\tVBP\t_\t1\tobj\t_\t_ + + """; + + /** The expected block the error listener prints for {@link #REVERSED_SENTENCE}. */ + static final String EXPECTED_REVERSED = "Expected: {\n" + + "1\tdogs\tNOUN\t0\troot" + System.lineSeparator() + + "2\trun\tVERB\t1\tobj" + System.lineSeparator() + "}"; + + /** The predicted block the error listener prints for the parse of {@link #SENTENCE}. */ + static final String PREDICTED_SENTENCE = "Predicted: {\n" + + "1\tdogs\tNOUN\t2\tnsubj" + System.lineSeparator() + + "2\trun\tVERB\t0\troot" + System.lineSeparator() + "}"; + + /** The CoNLL-U output of the parser tools for {@code dogs_NOUN run_VERB}. */ + static final String PARSED_SENTENCE = "1\tdogs\t_\t_\tNOUN\t_\t2\tnsubj\t_\t_\n" + + "2\trun\t_\t_\tVERB\t_\t0\troot\t_\t_\n\n"; + + /** The scores logged for a parser that reproduces all 40 tokens of the data. */ + static final String PERFECT_SCORES = "Tokens: 40; UAS: 1.0; LAS: 1.0"; + + /** The punctuation-free scores logged for the same parser; the data has no punctuation. */ + static final String PERFECT_SCORES_EXCLUDING_PUNCTUATION = + "Tokens excluding punctuation: 40; UAS: 1.0; LAS: 1.0"; + + /** The message of the CoNLL-U format for an encoding other than UTF-8. */ + static final String UTF_8_REQUIRED = "CoNLL-U data must use UTF-8"; + + /** + * Runs a tool while capturing the INFO messages its logger emits. + * + * @param tool The tool class whose logger is captured. + * @param run Runs the tool. + * @return The formatted log messages, in order. Never {@code null}. + */ + static List<String> logOf(Class<?> tool, Runnable run) { + final Logger logger = (Logger) LoggerFactory.getLogger(tool); + final Level previous = logger.getLevel(); + final ListAppender<ILoggingEvent> appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + logger.setLevel(Level.INFO); + try { + run.run(); + } finally { + logger.setLevel(previous); + logger.detachAppender(appender); + } + return appender.list.stream().map(ILoggingEvent::getFormattedMessage).toList(); + } +} diff --git a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserToolsTest.java b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserToolsTest.java new file mode 100644 index 000000000..f964823ff --- /dev/null +++ b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/FeedforwardDependencyParserToolsTest.java @@ -0,0 +1,526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.cmdline.CLI; +import opennlp.tools.cmdline.TerminateToolException; +import opennlp.tools.depparse.FeedforwardDependencyModel; +import opennlp.tools.depparse.FeedforwardDependencyParser; +import opennlp.tools.depparse.FeedforwardDependencyTrainer; +import opennlp.tools.ml.BeamSearch; +import opennlp.tools.util.model.BaseModel; + +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.EXPECTED_REVERSED; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.GOLD; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PARSED_SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PERFECT_SCORES; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PERFECT_SCORES_EXCLUDING_PUNCTUATION; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.PREDICTED_SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.REPETITIONS; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.REVERSED_SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.SENTENCE; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.UTF_8_REQUIRED; +import static opennlp.tools.cmdline.depparse.DependencyToolTestSupport.logOf; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Exercises the feedforward dependency parser tools: registration, training, refinement, inference. */ +public class FeedforwardDependencyParserToolsTest { + + /** + * The training parameters of a small network that learns + * {@link DependencyToolTestSupport#SENTENCE} quickly. + */ + private static final String TRAINING_PARAMETERS = """ + EmbeddingSize=16 + HiddenSize=32 + Iterations=30 + BatchSize=32 + LearningRate=0.1 + L2=0 + Dropout=0 + Cutoff=1 + """; + + /** The refinement parameters: one pass with a beam of three. */ + private static final String REFINE_PARAMETERS = """ + Iterations=1 + LearningRate=0.01 + BeamSize=3 + """; + + @TempDir + private Path dir; + + private Path data; + private Path model; + private Path trainingParameters; + private Path refineParameters; + + @BeforeEach + void writeData() throws IOException { + data = dir.resolve("train.conllu"); + model = dir.resolve("parser.bin"); + trainingParameters = dir.resolve("train.params"); + refineParameters = dir.resolve("refine.params"); + Files.writeString(data, SENTENCE.repeat(REPETITIONS)); + Files.writeString(trainingParameters, TRAINING_PARAMETERS); + Files.writeString(refineParameters, REFINE_PARAMETERS); + } + + /** + * Runs the trainer tool with the small training parameters plus extra arguments. + * + * @param extra Arguments appended after the common ones. + */ + private void train(String... extra) { + final List<String> args = new ArrayList<>(List.of("-lang", "eng", "-data", data.toString(), + "-model", model.toString(), "-params", trainingParameters.toString())); + args.addAll(List.of(extra)); + new FeedforwardDependencyParserTrainerTool().run("conllu", args.toArray(new String[0])); + } + + /** + * Runs the parser tool on the given standard input. + * + * @param input The tagged sentences to parse. + * @param args The tool arguments. + * @return The standard output of the tool, with line breaks normalized to {@code \n}. + */ + private static String parse(String input, String... args) { + final InputStream previousIn = System.in; + final PrintStream previousOut = System.out; + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + try { + System.setIn(new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))); + System.setOut(new PrintStream(output, true, StandardCharsets.UTF_8)); + new FeedforwardDependencyParserTool().run(args); + } finally { + System.setIn(previousIn); + System.setOut(previousOut); + } + return output.toString(StandardCharsets.UTF_8).replace("\r\n", "\n"); + } + + /** The three feedforward tools are registered with the command line. */ + @Test + void testToolsAreRegistered() { + assertTrue(CLI.getToolNames().contains("FeedforwardDependencyParser")); + assertTrue(CLI.getToolNames().contains("FeedforwardDependencyParserTrainer")); + assertTrue(CLI.getToolNames().contains("FeedforwardDependencyParserEvaluator")); + } + + /** No feedforward cross validator exists in the API, so none is registered. */ + @Test + void testNoCrossValidatorIsRegistered() { + assertFalse(CLI.getToolNames().contains("FeedforwardDependencyParserCrossValidator")); + } + + /** + * The trainer writes a model that loads, parses the training sentence, and records the + * training parameters. + */ + @Test + void testTrainerWritesALoadableModel() throws IOException { + train(); + final FeedforwardDependencyModel loaded = new FeedforwardDependencyModel(model); + assertEquals("eng", loaded.getLanguage()); + assertEquals(GOLD, new FeedforwardDependencyParser(loaded) + .parse(new String[] {"dogs", "run"}, new String[] {"NOUN", "VERB"})); + assertEquals("30", loaded.getManifestProperty(BaseModel.TRAINING_ITERATIONS_PROPERTY)); + assertEquals("1", loaded.getManifestProperty(BaseModel.TRAINING_CUTOFF_PROPERTY)); + assertNull(loaded.getManifestProperty(BeamSearch.BEAM_SIZE_PARAMETER)); + } + + /** + * Without {@code -params} the trainer's own defaults apply, not the maxent defaults + * of {@code TrainingParameters.defaultParams()}. + */ + @Test + void testTrainerWithoutParamsUsesTheTrainerDefaults() throws IOException { + new FeedforwardDependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "eng", "-data", data.toString(), "-model", model.toString()}); + final FeedforwardDependencyModel loaded = new FeedforwardDependencyModel(model); + assertEquals(Integer.toString(FeedforwardDependencyTrainer.DEFAULT_EPOCHS), + loaded.getManifestProperty(BaseModel.TRAINING_ITERATIONS_PROPERTY)); + assertEquals(Integer.toString(FeedforwardDependencyTrainer.DEFAULT_WORD_CUTOFF), + loaded.getManifestProperty(BaseModel.TRAINING_CUTOFF_PROPERTY)); + } + + /** With {@code -refineParams} the trained model is refined and records the refinement beam size. */ + @Test + void testTrainerRefinesWithTheRefineParameters() throws IOException { + train("-refineParams", refineParameters.toString()); + final FeedforwardDependencyModel loaded = new FeedforwardDependencyModel(model); + assertEquals("3", loaded.getManifestProperty(BeamSearch.BEAM_SIZE_PARAMETER)); + assertEquals("1", loaded.getManifestProperty(BaseModel.TRAINING_ITERATIONS_PROPERTY)); + assertEquals(GOLD, new FeedforwardDependencyParser(loaded, 3) + .parse(new String[] {"dogs", "run"}, new String[] {"NOUN", "VERB"})); + } + + /** With {@code -baseModel} the trainer refines a saved model instead of training one. */ + @Test + void testTrainerRefinesAnExistingModel() throws IOException { + train(); + final Path refined = dir.resolve("refined.bin"); + new FeedforwardDependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "eng", "-data", data.toString(), "-model", refined.toString(), + "-baseModel", model.toString(), "-refineParams", refineParameters.toString()}); + final FeedforwardDependencyModel loaded = new FeedforwardDependencyModel(refined); + assertEquals("3", loaded.getManifestProperty(BeamSearch.BEAM_SIZE_PARAMETER)); + assertEquals(GOLD, new FeedforwardDependencyParser(loaded, 3) + .parse(new String[] {"dogs", "run"}, new String[] {"NOUN", "VERB"})); + } + + /** {@code -baseModel} without {@code -refineParams} is rejected as an argument error. */ + @Test + void testTrainerRejectsBaseModelWithoutRefineParams() { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "eng", "-data", data.toString(), "-model", dir.resolve("out.bin").toString(), + "-baseModel", model.toString()})); + assertEquals(1, exception.getCode()); + assertEquals("-baseModel requires -refineParams", exception.getMessage()); + } + + /** {@code -params} together with {@code -baseModel} is rejected, because no training runs. */ + @Test + void testTrainerRejectsBaseModelWithParams() { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "eng", "-data", data.toString(), "-model", dir.resolve("out.bin").toString(), + "-baseModel", model.toString(), "-refineParams", refineParameters.toString(), + "-params", trainingParameters.toString()})); + assertEquals(1, exception.getCode()); + assertEquals("-params does not apply with -baseModel; refinement reads -refineParams", + exception.getMessage()); + } + + /** A {@code -lang} that differs from the language of the base model is rejected. */ + @Test + void testTrainerRejectsBaseModelOfAnotherLanguage() { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "spa", "-data", data.toString(), "-model", dir.resolve("out.bin").toString(), + "-baseModel", model.toString(), "-refineParams", refineParameters.toString()})); + assertEquals(1, exception.getCode()); + assertEquals("-lang spa does not match the base model language eng", exception.getMessage()); + } + + /** Refinement needs a beam of at least two; the trainer's message ends the tool. */ + @Test + void testTrainerRejectsRefinementBeamOfOne() throws IOException { + Files.writeString(refineParameters, "BeamSize=1\n"); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> train("-refineParams", refineParameters.toString())); + assertEquals(-1, exception.getCode()); + assertEquals("BeamSize must be at least 2: 1", exception.getMessage()); + assertFalse(Files.exists(model)); + } + + /** An out-of-range hyperparameter ends the tool with the trainer's message. */ + @Test + void testTrainerRejectsInvalidHyperparameter() throws IOException { + Files.writeString(trainingParameters, "HiddenSize=0\n"); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + this::train); + assertEquals(-1, exception.getCode()); + assertEquals("HiddenSize must be positive", exception.getMessage()); + assertFalse(Files.exists(model)); + } + + /** Data without any trainable sample ends the tool with the trainer's message. */ + @Test + void testTrainerOnEmptyDataTerminates() throws IOException { + Files.writeString(data, ""); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + this::train); + assertEquals(-1, exception.getCode()); + assertFalse(Files.exists(model)); + } + + /** The trainer passes an {@code -encoding} other than UTF-8 on to the format, which rejects it. */ + @Test + void testTrainerRejectsOtherEncodings() { + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> train("-encoding", "ISO-8859-1")); + assertEquals(-1, exception.getCode()); + assertEquals(UTF_8_REQUIRED, exception.getMessage()); + } + + /** The parser tool prints the greedy parse of a tagged line as CoNLL-U. */ + @Test + void testParserToolPrintsConllu() { + train(); + assertEquals(PARSED_SENTENCE, parse("dogs_NOUN run_VERB\n", model.toString())); + } + + /** With {@code -beamSize} the parser tool parses every non-blank line and skips blank ones. */ + @Test + void testParserToolParsesWithABeam() { + train(); + assertEquals(PARSED_SENTENCE + PARSED_SENTENCE, + parse("dogs_NOUN run_VERB\n\n \ndogs_NOUN run_VERB\n", "-beamSize", "4", model.toString())); + } + + /** + * @return Argument lists the parser tool does not accept. + */ + static Stream<Arguments> wrongParserArguments() { + return Stream.of( + Arguments.of((Object) new String[0]), + Arguments.of((Object) new String[] {"-beamSize", "2"}), + Arguments.of((Object) new String[] {"-beamSize", "2", "a", "b"}), + Arguments.of((Object) new String[] {"-other", "2", "model"}), + Arguments.of((Object) new String[] {"model", "-beamSize", "2"})); + } + + /** Argument lists other than {@code model} or {@code -beamSize n model} end the tool with its help text. */ + @ParameterizedTest(name = "args = {0}") + @MethodSource("wrongParserArguments") + void testParserToolRejectsWrongArguments(String[] args) { + final FeedforwardDependencyParserTool tool = new FeedforwardDependencyParserTool(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> tool.run(args)); + assertEquals(1, exception.getCode()); + assertEquals(tool.getHelp(), exception.getMessage()); + } + + /** A {@code -beamSize} value that is not a decimal integer is rejected as an argument error. */ + @ParameterizedTest(name = "beamSize = \"{0}\"") + @ValueSource(strings = {"abc", "2.5", "", " 2"}) + void testParserToolRejectsNonIntegerBeamSize(String beamSize) { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyParserTool().run(new String[] { + "-beamSize", beamSize, model.toString()})); + assertEquals(1, exception.getCode()); + assertEquals("-beamSize must be an integer: " + beamSize, exception.getMessage()); + } + + /** A {@code -beamSize} below one ends the tool with the parser's message. */ + @ParameterizedTest(name = "beamSize = {0}") + @ValueSource(ints = {Integer.MIN_VALUE, -1, 0}) + void testParserToolRejectsBeamSizeBelowOne(int beamSize) { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyParserTool().run(new String[] { + "-beamSize", Integer.toString(beamSize), model.toString()})); + assertEquals(-1, exception.getCode()); + assertEquals("beamSize must be positive", exception.getMessage()); + } + + /** Without {@code -beamSize} the tools parse with the beam a refined model records. */ + @Test + void testDefaultBeamSizeComesFromTheModel() throws IOException { + train(); + assertEquals(1, FeedforwardDependencyParserTool.beamSize( + new FeedforwardDependencyModel(model), null)); + train("-refineParams", refineParameters.toString()); + final FeedforwardDependencyModel refined = new FeedforwardDependencyModel(model); + assertEquals(3, FeedforwardDependencyParserTool.beamSize(refined, null)); + assertEquals(5, FeedforwardDependencyParserTool.beamSize(refined, 5)); + } + + /** The evaluator logs perfect greedy scores on the training data. */ + @Test + void testEvaluatorLogsAttachmentScores() { + train(); + final List<String> log = logOf(FeedforwardDependencyParserEvaluatorTool.class, + () -> new FeedforwardDependencyParserEvaluatorTool().run("conllu", new String[] { + "-model", model.toString(), "-data", data.toString()})); + assertEquals(List.of(PERFECT_SCORES, PERFECT_SCORES_EXCLUDING_PUNCTUATION), log); + } + + /** The evaluator logs the same scores when it parses with a beam. */ + @Test + void testEvaluatorLogsAttachmentScoresWithABeam() { + train(); + final List<String> log = logOf(FeedforwardDependencyParserEvaluatorTool.class, + () -> new FeedforwardDependencyParserEvaluatorTool().run("conllu", new String[] { + "-model", model.toString(), "-data", data.toString(), "-beamSize", "4"})); + assertEquals(List.of(PERFECT_SCORES, PERFECT_SCORES_EXCLUDING_PUNCTUATION), log); + } + + /** A {@code -beamSize} below one ends the evaluator with the parser's message. */ + @Test + void testEvaluatorRejectsBeamSizeBelowOne() { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyParserEvaluatorTool().run("conllu", new String[] { + "-model", model.toString(), "-data", data.toString(), "-beamSize", "0"})); + assertEquals(-1, exception.getCode()); + assertEquals("beamSize must be positive", exception.getMessage()); + } + + /** With {@code -misclassified true} the evaluator prints a misparsed sentence next to its gold tree. */ + @Test + void testEvaluatorPrintsMisparsedSamples() throws IOException { + train(); + final Path test = dir.resolve("test.conllu"); + Files.writeString(test, REVERSED_SENTENCE); + final List<String> log = logOf(DependencyEvaluationErrorListener.class, + () -> new FeedforwardDependencyParserEvaluatorTool().run("conllu", new String[] { + "-model", model.toString(), "-data", test.toString(), "-misclassified", "true"})); + assertEquals(List.of(EXPECTED_REVERSED + "\n" + PREDICTED_SENTENCE), log); + } + + /** Without {@code -misclassified} the evaluator prints no samples. */ + @Test + void testEvaluatorWithoutMisclassifiedFlagPrintsNoSamples() throws IOException { + train(); + final Path test = dir.resolve("test.conllu"); + Files.writeString(test, REVERSED_SENTENCE); + final List<String> log = logOf(DependencyEvaluationErrorListener.class, + () -> new FeedforwardDependencyParserEvaluatorTool().run("conllu", new String[] { + "-model", model.toString(), "-data", test.toString()})); + assertEquals(List.of(), log); + } + + /** The feedforward loader rejects a maxent dependency model as an invalid format. */ + @Test + void testModelLoaderRejectsAMaxentModel() { + new DependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "eng", "-data", data.toString(), "-model", model.toString()}); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyModelLoader().load(model.toFile())); + assertEquals(-1, exception.getCode()); + assertEquals("Model has invalid format", exception.getMessage()); + } + + /** The maxent loader rejects a feedforward model the same way, through the shared loader. */ + @Test + void testMaxentModelLoaderRejectsAFeedforwardModel() { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new DependencyModelLoader().load(model.toFile())); + assertEquals(-1, exception.getCode()); + assertEquals("Model has invalid format", exception.getMessage()); + } + + /** The feedforward loader loads a model the trainer wrote. */ + @Test + void testModelLoaderLoadsAFeedforwardModel() { + train(); + assertEquals("eng", new FeedforwardDependencyModelLoader().load(model.toFile()).getLanguage()); + } + + /** A {@code -params} file without {@code Iterations} trains for the trainer's default epochs. */ + @Test + void testTrainerParamsWithoutIterationsUseTheTrainerDefault() throws IOException { + Files.writeString(trainingParameters, "EmbeddingSize=16\nHiddenSize=32\nDropout=0\nCutoff=1\n"); + train(); + assertEquals(Integer.toString(FeedforwardDependencyTrainer.DEFAULT_EPOCHS), + new FeedforwardDependencyModel(model).getManifestProperty( + BaseModel.TRAINING_ITERATIONS_PROPERTY)); + } + + /** A missing {@code -refineParams} file ends the tool before training and writes no model. */ + @Test + void testTrainerRejectsMissingRefineParamsFile() { + final Path missing = dir.resolve("missing.params"); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> train("-refineParams", missing.toString())); + assertEquals(-1, exception.getCode()); + assertEquals("The Training Parameter file does not exist! Path: " + missing.toAbsolutePath(), + exception.getMessage()); + assertFalse(Files.exists(model)); + } + + /** A maxent model given as {@code -baseModel} ends the trainer through the shared loader. */ + @Test + void testTrainerRejectsMaxentBaseModel() { + new DependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "eng", "-data", data.toString(), "-model", model.toString()}); + final Path out = dir.resolve("out.bin"); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> new FeedforwardDependencyParserTrainerTool().run("conllu", new String[] { + "-lang", "eng", "-data", data.toString(), "-model", out.toString(), + "-baseModel", model.toString(), "-refineParams", refineParameters.toString()})); + assertEquals(-1, exception.getCode()); + assertEquals("Model has invalid format", exception.getMessage()); + assertFalse(Files.exists(out)); + } + + /** Empty standard input produces no output. */ + @Test + void testParserToolOnEmptyInputPrintsNothing() { + train(); + assertEquals("", parse("", model.toString())); + } + + /** A token without a tag separator ends the parser tool with the standard input error. */ + @Test + void testParserToolRejectsTokenWithoutTag() { + train(); + final TerminateToolException exception = assertThrows(TerminateToolException.class, + () -> parse("dogs run_VERB\n", model.toString())); + assertEquals(-1, exception.getCode()); + assertEquals("IO Error while reading from stdin: Cannot find \"_\" inside token 'dogs'!", + exception.getMessage()); + } + + /** The help text names {@code -beamSize} and its default, so a change to it is deliberate. */ + @Test + void testParserToolHelp() { + assertEquals("Usage: " + CLI.CMD + " FeedforwardDependencyParser [-beamSize n] model" + + " < tagged-sentences\n" + + "-beamSize n: keep n transition sequences per step. Defaults to the beam size the" + + " model was refined with, or 1 (greedy) for a model that was not refined.", + new FeedforwardDependencyParserTool().getHelp()); + } + + /** With a beam, {@code -misclassified true} prints the same misparsed sentence as greedy decoding. */ + @Test + void testEvaluatorPrintsMisparsedSamplesWithABeam() throws IOException { + train(); + final Path test = dir.resolve("test.conllu"); + Files.writeString(test, REVERSED_SENTENCE); + final List<String> log = logOf(DependencyEvaluationErrorListener.class, + () -> new FeedforwardDependencyParserEvaluatorTool().run("conllu", new String[] { + "-model", model.toString(), "-data", test.toString(), "-misclassified", "true", + "-beamSize", "4"})); + assertEquals(List.of(EXPECTED_REVERSED + "\n" + PREDICTED_SENTENCE), log); + } +} diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml index 13b25c939..0bf03b209 100644 --- a/opennlp-docs/src/docbkx/dependency.xml +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -117,6 +117,33 @@ DependencyParser parser = new FeedforwardDependencyParser( sequences per step and scores each of them. One parser instance can be shared between threads. </para> + + <section xml:id="tools.dependency.feedforward.cli"> + <title>Command-line tools</title> + <para> + The feedforward tools read the same UTF-8 CoNLL-U input, take the same + <code>-tagset</code> option, and write the same CoNLL-U and score output as the + tools in <xref linkend="tools.dependency.cli"/>. The <code>-params</code> file holds + the <code>FeedforwardDependencyTrainer</code> keys, for example + <code>EmbeddingSize</code>, <code>HiddenSize</code>, <code>Iterations</code>, and + <code>Dropout</code>; absent keys, or a missing file, take the trainer's defaults. + A <code>-refineParams</code> file refines the trained model with beam search, with + <code>BeamSize</code> (at least two) setting the beam. With <code>-baseModel</code> + the trainer refines an existing model instead of training a new one; its + <code>-lang</code> must match the model. For example: + </para> + <programlisting> +<![CDATA[opennlp FeedforwardDependencyParserTrainer.conllu -lang eng -data train.conllu \ + -model en-depparse-ff.bin -params ff.params -refineParams refine.params +opennlp FeedforwardDependencyParserEvaluator.conllu -model en-depparse-ff.bin -data dev.conllu +opennlp FeedforwardDependencyParser -beamSize 4 en-depparse-ff.bin < tagged-sentences.txt]]> + </programlisting> + <para> + The parser and evaluator take an optional <code>-beamSize</code>. Without it they + use the beam size a refined model records, and parse a model that was not refined + greedily. There is no feedforward cross-validation tool. + </para> + </section> </section> <section xml:id="tools.dependency.parsing">
