Repository: opennlp Updated Branches: refs/heads/parser_regression 3ac2fb377 -> 158624265 (forced update)
OPENNLP-973: Respect setting of PrintMessages in DataIndexer impls Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/b90712e7 Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/b90712e7 Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/b90712e7 Branch: refs/heads/parser_regression Commit: b90712e74cce74cca3ff044e630e47f58e774b62 Parents: 8fc46b6 Author: Jörn Kottmann <[email protected]> Authored: Wed Feb 1 15:38:19 2017 +0100 Committer: Jörn Kottmann <[email protected]> Committed: Thu Apr 20 12:40:17 2017 +0200 ---------------------------------------------------------------------- .../tools/ml/model/AbstractDataIndexer.java | 2 +- .../tools/ml/model/OnePassDataIndexer.java | 18 ++++++++--------- .../tools/ml/model/TwoPassDataIndexer.java | 21 ++++++++++---------- 3 files changed, 21 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/opennlp/blob/b90712e7/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java index 0eabf87..be6a6e4 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java @@ -133,7 +133,7 @@ public abstract class AbstractDataIndexer implements DataIndexer { throw new InsufficientTrainingDataException("Insufficient training data to create model."); } - if (sort) System.out.println("done. Reduced " + numEvents + " events to " + numUniqueEvents + "."); + if (sort) display("done. Reduced " + numEvents + " events to " + numUniqueEvents + ".\n"); contexts = new int[numUniqueEvents][]; outcomeList = new int[numUniqueEvents]; http://git-wip-us.apache.org/repos/asf/opennlp/blob/b90712e7/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java index 7b53251..cf8fb7f 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java @@ -48,24 +48,24 @@ public class OnePassDataIndexer extends AbstractDataIndexer { List<Event> events; List<ComparableEvent> eventsToCompare; - System.out.println("Indexing events using cutoff of " + cutoff + "\n"); + display("Indexing events using cutoff of " + cutoff + "\n\n"); - System.out.print("\tComputing event counts... "); + display("\tComputing event counts... "); events = computeEventCounts(eventStream, predicateIndex, cutoff); - System.out.println("done. " + events.size() + " events"); + display("done. " + events.size() + " events\n"); - System.out.print("\tIndexing... "); + display("\tIndexing... "); eventsToCompare = index(events, predicateIndex); // done with event list events = null; // done with predicates predicateIndex = null; - System.out.println("done."); + display("done.\n"); - System.out.print("Sorting and merging events... "); + display("Sorting and merging events... "); sortAndMerge(eventsToCompare, sort); - System.out.println("Done indexing."); + display("Done indexing.\n"); } /** @@ -140,8 +140,8 @@ public class OnePassDataIndexer extends AbstractDataIndexer { ce = new ComparableEvent(ocID, cons); eventsToCompare.add(ce); } else { - System.err.println("Dropped event " + ev.getOutcome() + ":" - + Arrays.asList(ev.getContext())); + display("Dropped event " + ev.getOutcome() + ":" + + Arrays.asList(ev.getContext()) + "\n"); } // recycle the TIntArrayList indexedContext.clear(); http://git-wip-us.apache.org/repos/asf/opennlp/blob/b90712e7/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java index 133c350..b3cc89b 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java @@ -55,17 +55,17 @@ public class TwoPassDataIndexer extends AbstractDataIndexer { Map<String,Integer> predicateIndex = new HashMap<>(); List<ComparableEvent> eventsToCompare; - System.out.println("Indexing events using cutoff of " + cutoff + "\n"); + display("Indexing events using cutoff of " + cutoff + "\n\n"); - System.out.print("\tComputing event counts... "); + display("\tComputing event counts... "); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); Writer osw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmp),"UTF8")); int numEvents = computeEventCounts(eventStream, osw, predicateIndex, cutoff); - System.out.println("done. " + numEvents + " events"); + display("done. " + numEvents + " events\n"); - System.out.print("\tIndexing... "); + display("\tIndexing... "); try (FileEventStream fes = new FileEventStream(tmp)) { eventsToCompare = index(numEvents, fes, predicateIndex); @@ -73,16 +73,16 @@ public class TwoPassDataIndexer extends AbstractDataIndexer { // done with predicates predicateIndex = null; tmp.delete(); - System.out.println("done."); + display("done.\n"); if (sort) { - System.out.print("Sorting and merging events... "); + display("Sorting and merging events... "); } else { - System.out.print("Collecting events... "); + display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); - System.out.println("Done indexing."); + display("Done indexing.\n"); } /** @@ -120,9 +120,11 @@ public class TwoPassDataIndexer extends AbstractDataIndexer { return eventCount; } + // TODO: merge this code with the copy and paste version in OnePassDataIndexer private List<ComparableEvent> index(int numEvents, ObjectStream<Event> es, Map<String,Integer> predicateIndex) throws IOException { Map<String,Integer> omap = new HashMap<>(); + int outcomeCount = 0; List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents); List<Integer> indexedContext = new ArrayList<>(); @@ -159,7 +161,7 @@ public class TwoPassDataIndexer extends AbstractDataIndexer { eventsToCompare.add(ce); } else { - System.err.println("Dropped event " + ev.getOutcome() + ":" + Arrays.asList(ev.getContext())); + display("Dropped event " + ev.getOutcome() + ":" + Arrays.asList(ev.getContext()) + "\n"); } // recycle the TIntArrayList indexedContext.clear(); @@ -168,6 +170,5 @@ public class TwoPassDataIndexer extends AbstractDataIndexer { predLabels = toIndexedStringArray(predicateIndex); return eventsToCompare; } - }
