Repository: opennlp Updated Branches: refs/heads/master 560c48438 -> c2097051c
OPENNLP-1094: Remove pmap indirection Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/c2097051 Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/c2097051 Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/c2097051 Branch: refs/heads/master Commit: c2097051c11479284f73485da46d02ba15e241a3 Parents: 560c484 Author: Jörn Kottmann <[email protected]> Authored: Wed Jun 14 14:34:14 2017 +0200 Committer: Jörn Kottmann <[email protected]> Committed: Fri Jun 16 14:43:18 2017 +0200 ---------------------------------------------------------------------- .../java/opennlp/tools/ml/maxent/GISModel.java | 39 +++++++++++++++++--- .../tools/ml/maxent/io/GISModelWriter.java | 14 ++++--- .../tools/ml/maxent/quasinewton/QNModel.java | 11 +++--- .../opennlp/tools/ml/model/AbstractModel.java | 23 ++++-------- .../main/java/opennlp/tools/ml/model/Prior.java | 9 +++++ .../opennlp/tools/ml/model/UniformPrior.java | 5 +++ .../tools/ml/naivebayes/NaiveBayesModel.java | 26 ++++++++----- .../ml/naivebayes/NaiveBayesModelWriter.java | 13 +++++-- .../tools/ml/perceptron/PerceptronModel.java | 35 +++++++++--------- .../ml/perceptron/PerceptronModelWriter.java | 15 +++++--- .../SimplePerceptronSequenceTrainer.java | 13 ++----- 11 files changed, 125 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java index 81b2690..700a81d 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java @@ -103,10 +103,9 @@ public final class GISModel extends AbstractModel { * method getOutcome(int i). */ public final double[] eval(String[] context, float[] values, double[] outsums) { - int[] scontexts = new int[context.length]; + Context[] scontexts = new Context[context.length]; for (int i = 0; i < context.length; i++) { - Integer ci = pmap.get(context[i]); - scontexts[i] = ci == null ? -1 : ci; + scontexts[i] = pmap.get(context[i]); } prior.logPrior(outsums, scontexts, values); return GISModel.eval(scontexts, values, outsums, evalParams); @@ -154,14 +153,42 @@ public final class GISModel extends AbstractModel { */ static double[] eval(int[] context, float[] values, double[] prior, EvalParameters model) { - Context[] params = model.getParams(); + + Context[] scontexts = new Context[context.length]; + for (int i = 0; i < context.length; i++) { + scontexts[i] = model.getParams()[context[i]]; + } + + return GISModel.eval(scontexts, values, prior, model); + } + + /** + * Use this model to evaluate a context and return an array of the likelihood + * of each outcome given the specified context and the specified parameters. + * + * @param context + * The integer values of the predicates which have been observed at + * the present decision point. + * @param values + * The values for each of the parameters. + * @param prior + * The prior distribution for the specified context. + * @param model + * The set of parametes used in this computation. + * @return The normalized probabilities for the outcomes given the context. + * The indexes of the double[] are the outcome ids, and the actual + * string representation of the outcomes can be obtained from the + * method getOutcome(int i). + */ + static double[] eval(Context[] context, float[] values, double[] prior, + EvalParameters model) { int[] numfeats = new int[model.getNumOutcomes()]; int[] activeOutcomes; double[] activeParameters; double value = 1; for (int ci = 0; ci < context.length; ci++) { - if (context[ci] >= 0) { - Context predParams = params[context[ci]]; + if (context[ci] != null) { + Context predParams = context[ci]; activeOutcomes = predParams.getOutcomes(); activeParameters = predParams.getParameters(); if (values != null) { http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java index e11dcba..3fdeaf0 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java @@ -42,14 +42,18 @@ public abstract class GISModelWriter extends AbstractModelWriter { Object[] data = model.getDataStructures(); - PARAMS = (Context[]) data[0]; - @SuppressWarnings("unchecked") - Map<String, Integer> pmap = (Map<String, Integer>) data[1]; + Map<String, Context> pmap = (Map<String, Context>) data[1]; + OUTCOME_LABELS = (String[]) data[2]; + PARAMS = new Context[pmap.size()]; PRED_LABELS = new String[pmap.size()]; - for (String pred : pmap.keySet()) { - PRED_LABELS[pmap.get(pred)] = pred; + + int i = 0; + for (Map.Entry<String, Context> pred : pmap.entrySet()) { + PRED_LABELS[i] = pred.getKey(); + PARAMS[i] = pred.getValue(); + i++; } } http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNModel.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNModel.java index d73d6cc..85f6dc3 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNModel.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNModel.java @@ -31,7 +31,7 @@ public class QNModel extends AbstractModel { return this.outcomeNames.length; } - private Integer getPredIndex(String predicate) { + private Context getPredIndex(String predicate) { return pmap.get(predicate); } @@ -60,17 +60,16 @@ public class QNModel extends AbstractModel { * @return Normalized probabilities for the outcomes given the context. */ private double[] eval(String[] context, float[] values, double[] probs) { - Context[] params = evalParams.getParams(); for (int ci = 0; ci < context.length; ci++) { - Integer predIdx = getPredIndex(context[ci]); + Context pred = getPredIndex(context[ci]); - if (predIdx != null) { + if (pred != null) { double predValue = 1.0; if (values != null) predValue = values[ci]; - double[] parameters = params[predIdx].getParameters(); - int[] outcomes = params[predIdx].getOutcomes(); + double[] parameters = pred.getParameters(); + int[] outcomes = pred.getOutcomes(); for (int i = 0; i < outcomes.length; i++) { int oi = outcomes[i]; probs[oi] += predValue * parameters[i]; http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java index eb80f1b..4783a71 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java @@ -26,7 +26,7 @@ import java.util.Objects; public abstract class AbstractModel implements MaxentModel { /** Mapping between predicates/contexts and an integer representing them. */ - protected Map<String, Integer> pmap; + protected Map<String, Context> pmap; /** The names of the outcomes. */ protected String[] outcomeNames; /** Parameters for the model. */ @@ -39,32 +39,23 @@ public abstract class AbstractModel implements MaxentModel { /** The type of the model. */ protected ModelType modelType; - /** - * @deprecated this will be removed in 1.8.1, pmap should be private - * - * @param params - * @param predLabels - * @param pmap - * @param outcomeNames - */ - @Deprecated - public AbstractModel(Context[] params, String[] predLabels, - Map<String, Integer> pmap, String[] outcomeNames) { + protected AbstractModel(Context[] params, String[] predLabels, + Map<String, Context> pmap, String[] outcomeNames) { this.pmap = pmap; this.outcomeNames = outcomeNames; this.evalParams = new EvalParameters(params,outcomeNames.length); } public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames) { - init(predLabels, outcomeNames); + init(predLabels, params, outcomeNames); this.evalParams = new EvalParameters(params, outcomeNames.length); } - private void init(String[] predLabels, String[] outcomeNames) { + private void init(String[] predLabels, Context[] params, String[] outcomeNames) { this.pmap = new HashMap<>(predLabels.length); for (int i = 0; i < predLabels.length; i++) { - pmap.put(predLabels[i], i); + pmap.put(predLabels[i], params[i]); } this.outcomeNames = outcomeNames; @@ -188,7 +179,7 @@ public abstract class AbstractModel implements MaxentModel { AbstractModel model = (AbstractModel) obj; return pmap.equals(model.pmap) && Objects.deepEquals(outcomeNames, model.outcomeNames) - && evalParams.equals(model.evalParams) && Objects.equals(prior, model.prior); + && Objects.equals(prior, model.prior); } return false; http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java index b0ce405..89ddc69 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java @@ -41,6 +41,15 @@ public interface Prior { void logPrior(double[] dist, int[] context, float[] values); /** + * Populates the specified array with the the log of the distribution for the specified context. + * The returned array will be overwritten and needs to be re-initialized with every call to this method. + * @param dist An array to be populated with the log of the prior distribution. + * @param context The indices of the contextual predicates for an event. + * @param values The values associated with the context. + */ + void logPrior(double[] dist, Context[] context, float[] values); + + /** * Method to specify the label for the outcomes and contexts. This is used to map * integer outcomes and contexts to their string values. This method is called prior * to any call to #logPrior. http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/model/UniformPrior.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/UniformPrior.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/UniformPrior.java index 412fdaf..17574e2 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/UniformPrior.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/UniformPrior.java @@ -33,6 +33,11 @@ public class UniformPrior implements Prior { } } + @Override + public void logPrior(double[] dist, Context[] context, float[] values) { + logPrior(dist, (int[]) null, values); + } + public void logPrior(double[] dist, int[] context) { logPrior(dist,context,null); } http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java index 98c2735..85e17eb 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java @@ -31,8 +31,7 @@ public class NaiveBayesModel extends AbstractModel { protected double[] outcomeTotals; protected long vocabulary; - @Deprecated - public NaiveBayesModel(Context[] params, String[] predLabels, Map<String, Integer> pmap, + NaiveBayesModel(Context[] params, String[] predLabels, Map<String, Context> pmap, String[] outcomeNames) { super(params, predLabels, pmap, outcomeNames); outcomeTotals = initOutcomeTotals(outcomeNames, params); @@ -75,11 +74,10 @@ public class NaiveBayesModel extends AbstractModel { } public double[] eval(String[] context, float[] values, double[] outsums) { - int[] scontexts = new int[context.length]; + Context[] scontexts = new Context[context.length]; java.util.Arrays.fill(outsums, 0); for (int i = 0; i < context.length; i++) { - Integer ci = pmap.get(context[i]); - scontexts[i] = ci == null ? -1 : ci; + scontexts[i] = pmap.get(context[i]); } return eval(scontexts, values, outsums, evalParams, true); } @@ -88,8 +86,8 @@ public class NaiveBayesModel extends AbstractModel { return eval(context, null, prior, model, true); } - static double[] eval(int[] context, float[] values, double[] prior, - EvalParameters model, boolean normalize) { + static double[] eval(Context[] context, float[] values, double[] prior, + EvalParameters model, boolean normalize) { Probabilities<Integer> probabilities = new LogProbabilities<>(); Context[] params = model.getParams(); double[] outcomeTotals = model instanceof NaiveBayesEvalParameters @@ -100,8 +98,8 @@ public class NaiveBayesModel extends AbstractModel { int[] activeOutcomes; double value = 1; for (int ci = 0; ci < context.length; ci++) { - if (context[ci] >= 0) { - Context predParams = params[context[ci]]; + if (context[ci] != null) { + Context predParams = context[ci]; activeOutcomes = predParams.getOutcomes(); activeParameters = predParams.getParameters(); if (values != null) { @@ -130,6 +128,16 @@ public class NaiveBayesModel extends AbstractModel { return prior; } + static double[] eval(int[] context, float[] values, double[] prior, + EvalParameters model, boolean normalize) { + Context[] scontexts = new Context[context.length]; + for (int i = 0; i < context.length; i++) { + scontexts[i] = model.getParams()[context[i]]; + } + + return eval(scontexts, values, prior, model, normalize); + } + private static double getProbability(double numerator, double denominator, double vocabulary, boolean isSmoothed) { if (isSmoothed) http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java index 510bf76..60efbaa 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java @@ -46,12 +46,17 @@ public abstract class NaiveBayesModelWriter extends AbstractModelWriter { PARAMS = (Context[]) data[0]; @SuppressWarnings("unchecked") - Map<String, Integer> pmap = (Map<String, Integer>) data[1]; - OUTCOME_LABELS = (String[]) data[2]; + Map<String, Context> pmap = (Map<String, Context>) data[1]; + OUTCOME_LABELS = (String[]) data[2]; + PARAMS = new Context[pmap.size()]; PRED_LABELS = new String[pmap.size()]; - for (String pred : pmap.keySet()) { - PRED_LABELS[pmap.get(pred)] = pred; + + int i = 0; + for (Map.Entry<String, Context> pred : pmap.entrySet()) { + PRED_LABELS[i] = pred.getKey(); + PARAMS[i] = pred.getValue(); + i++; } } http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModel.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModel.java index 39c4891..b931c50 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModel.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModel.java @@ -17,24 +17,12 @@ package opennlp.tools.ml.perceptron; -import java.util.Map; - import opennlp.tools.ml.model.AbstractModel; import opennlp.tools.ml.model.Context; import opennlp.tools.ml.model.EvalParameters; public class PerceptronModel extends AbstractModel { - /** - * @deprecated this will be removed in 1.8.1, pmap should be private - */ - @Deprecated - public PerceptronModel(Context[] params, String[] predLabels, Map<String, Integer> pmap, - String[] outcomeNames) { - super(params,predLabels,pmap,outcomeNames); - modelType = ModelType.Perceptron; - } - public PerceptronModel(Context[] params, String[] predLabels, String[] outcomeNames) { super(params,predLabels,outcomeNames); modelType = ModelType.Perceptron; @@ -53,11 +41,10 @@ public class PerceptronModel extends AbstractModel { } public double[] eval(String[] context, float[] values,double[] outsums) { - int[] scontexts = new int[context.length]; + Context[] scontexts = new Context[context.length]; java.util.Arrays.fill(outsums, 0); for (int i = 0; i < context.length; i++) { - Integer ci = pmap.get(context[i]); - scontexts[i] = ci == null ? -1 : ci; + scontexts[i] = pmap.get(context[i]); } return eval(scontexts,values,outsums,evalParams,true); } @@ -68,13 +55,23 @@ public class PerceptronModel extends AbstractModel { static double[] eval(int[] context, float[] values, double[] prior, EvalParameters model, boolean normalize) { + Context[] scontexts = new Context[context.length]; + for (int i = 0; i < context.length; i++) { + scontexts[i] = model.getParams()[context[i]]; + } + + return eval(scontexts, values, prior, model, normalize); + } + + static double[] eval(Context[] context, float[] values, double[] prior, EvalParameters model, + boolean normalize) { Context[] params = model.getParams(); double[] activeParameters; int[] activeOutcomes; double value = 1; for (int ci = 0; ci < context.length; ci++) { - if (context[ci] >= 0) { - Context predParams = params[context[ci]]; + if (context[ci] != null) { + Context predParams = context[ci]; activeOutcomes = predParams.getOutcomes(); activeParameters = predParams.getParameters(); if (values != null) { @@ -86,6 +83,7 @@ public class PerceptronModel extends AbstractModel { } } } + if (normalize) { int numOutcomes = model.getNumOutcomes(); @@ -102,8 +100,9 @@ public class PerceptronModel extends AbstractModel { normal += prior[oid]; } - for (int oid = 0; oid < numOutcomes; oid++) + for (int oid = 0; oid < numOutcomes; oid++) { prior[oid] /= normal; + } } return prior; } http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java index 133400b..93f4fcf 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java @@ -46,13 +46,18 @@ public abstract class PerceptronModelWriter extends AbstractModelWriter { this.numOutcomes = model.getNumOutcomes(); PARAMS = (Context[]) data[0]; - @SuppressWarnings("unchecked") - Map<String, Integer> pmap = (Map<String, Integer>) data[1]; - OUTCOME_LABELS = (String[])data[2]; + Map<String, Context> pmap = (Map<String, Context>) data[1]; + + OUTCOME_LABELS = (String[]) data[2]; + PARAMS = new Context[pmap.size()]; PRED_LABELS = new String[pmap.size()]; - for (String pred : pmap.keySet()) { - PRED_LABELS[pmap.get(pred)] = pred; + + int i = 0; + for (Map.Entry<String, Context> pred : pmap.entrySet()) { + PRED_LABELS[i] = pred.getKey(); + PARAMS[i] = pred.getValue(); + i++; } } http://git-wip-us.apache.org/repos/asf/opennlp/blob/c2097051/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java index a9ac516..b812836 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java @@ -193,12 +193,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT /* Create and return the model ****/ String[] updatedPredLabels = predLabels; - /* - String[] updatedPredLabels = new String[pmap.size()]; - for (String pred : pmap.keySet()) { - updatedPredLabels[pmap.get(pred)]=pred; - } - */ + if (useAverage) { return new PerceptronModel(averageParams, updatedPredLabels, outcomeLabels); } @@ -235,7 +230,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT for (int oi = 0; oi < numOutcomes; oi++) { featureCounts.add(new HashMap<>()); } - PerceptronModel model = new PerceptronModel(params,predLabels,pmap,outcomeLabels); + PerceptronModel model = new PerceptronModel(params,predLabels,outcomeLabels); sequenceStream.reset(); @@ -328,7 +323,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT } } } - model = new PerceptronModel(params,predLabels,pmap,outcomeLabels); + model = new PerceptronModel(params,predLabels,outcomeLabels); } si++; } @@ -364,7 +359,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT Sequence sequence; while ((sequence = sequenceStream.read()) != null) { Event[] taggerEvents = sequenceStream.updateContext(sequence, - new PerceptronModel(params,predLabels,pmap,outcomeLabels)); + new PerceptronModel(params,predLabels,outcomeLabels)); for (int ei = 0; ei < taggerEvents.length; ei++, oei++) { int max = omap.get(taggerEvents[ei].getOutcome()); if (max == outcomeList[oei]) {
