[17/50] [abbrv] opennlp git commit: OPENNLP-983: Make suffix/prefix length configurable

2017-04-20 Thread joern 
OPENNLP-983: Make suffix/prefix length configurable

This closes #121


Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/dc91b254
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/dc91b254
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/dc91b254

Branch: refs/heads/parser_regression
Commit: dc91b254adfd2bcb5aa4525a30162979f0e1fde3
Parents: 733d27d
Author: jzonthemtn 
Authored: Mon Feb 13 07:57:21 2017 -0500
Committer: Jörn Kottmann 
Committed: Thu Apr 20 12:40:20 2017 +0200

--
 .../tools/util/featuregen/GeneratorFactory.java | 22 -
 .../util/featuregen/PrefixFeatureGenerator.java | 32 +--
 .../util/featuregen/SuffixFeatureGenerator.java | 33 +--
 .../featuregen/PrefixFeatureGeneratorTest.java  | 92 
 .../featuregen/SuffixFeatureGeneratorTest.java  | 92 
 5 files changed, 251 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/opennlp/blob/dc91b254/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
index fa97f43..ef08cfb 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
@@ -555,7 +555,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new PrefixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int prefixLength = PrefixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+prefixLength = Integer.parseInt(attribute);
+  }
+
+  return new PrefixFeatureGenerator(prefixLength);
 }
 
 static void register(Map factoryMap) {
@@ -570,7 +579,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new SuffixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int suffixLength = SuffixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+suffixLength = Integer.parseInt(attribute);
+  }
+
+  return new SuffixFeatureGenerator(suffixLength);
 }
 
 static void register(Map factoryMap) {

http://git-wip-us.apache.org/repos/asf/opennlp/blob/dc91b254/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
index 8cdd48f..04fcd15 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
@@ -21,21 +21,35 @@ import java.util.List;
 
 public class PrefixFeatureGenerator implements AdaptiveFeatureGenerator {
 
-  private static final int PREFIX_LENGTH = 4;
-
-  private static String[] getPrefixes(String lex) {
-String[] prefs = new String[PREFIX_LENGTH];
-for (int li = 0; li < PREFIX_LENGTH; li++) {
-  prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
-}
-return prefs;
+  static final int DEFAULT_MAX_LENGTH = 4;
+  
+  private final int prefixLength;
+  
+  public PrefixFeatureGenerator() {
+prefixLength = DEFAULT_MAX_LENGTH;
+  }
+  
+  public PrefixFeatureGenerator(int prefixLength) {
+this.prefixLength = prefixLength;
   }
 
+  @Override
   public void createFeatures(List features, String[] tokens, int index,
   String[] previousOutcomes) {
-String[] prefs = PrefixFeatureGenerator.getPrefixes(tokens[index]);
+String[] prefs = getPrefixes(tokens[index]);
 for (String pref : prefs) {
   features.add("pre=" + pref);
 }
   }
+  
+  private String[] getPrefixes(String lex) {
+  
+int prefixes = Math.min(prefixLength, lex.length());
+
+String[] prefs = new String[prefixes];
+for (int li = 0; li < prefixes; li++) {
+  prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
+}
+return prefs;

[15/50] [abbrv] opennlp git commit: OPENNLP-983: Make suffix/prefix length configurable

2017-04-16 Thread joern 
OPENNLP-983: Make suffix/prefix length configurable

This closes #121


Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/1cd2658d
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/1cd2658d
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/1cd2658d

Branch: refs/heads/parser_regression
Commit: 1cd2658d0179afcf982229fab9c24da62f944c58
Parents: fc3b12f
Author: jzonthemtn 
Authored: Mon Feb 13 07:57:21 2017 -0500
Committer: Jörn Kottmann 
Committed: Sun Apr 16 19:24:53 2017 +0200

--
 .../tools/util/featuregen/GeneratorFactory.java | 22 -
 .../util/featuregen/PrefixFeatureGenerator.java | 32 +--
 .../util/featuregen/SuffixFeatureGenerator.java | 33 +--
 .../featuregen/PrefixFeatureGeneratorTest.java  | 92 
 .../featuregen/SuffixFeatureGeneratorTest.java  | 92 
 5 files changed, 251 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/opennlp/blob/1cd2658d/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
index fa97f43..ef08cfb 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
@@ -555,7 +555,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new PrefixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int prefixLength = PrefixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+prefixLength = Integer.parseInt(attribute);
+  }
+
+  return new PrefixFeatureGenerator(prefixLength);
 }
 
 static void register(Map factoryMap) {
@@ -570,7 +579,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new SuffixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int suffixLength = SuffixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+suffixLength = Integer.parseInt(attribute);
+  }
+
+  return new SuffixFeatureGenerator(suffixLength);
 }
 
 static void register(Map factoryMap) {

http://git-wip-us.apache.org/repos/asf/opennlp/blob/1cd2658d/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
index 8cdd48f..04fcd15 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
@@ -21,21 +21,35 @@ import java.util.List;
 
 public class PrefixFeatureGenerator implements AdaptiveFeatureGenerator {
 
-  private static final int PREFIX_LENGTH = 4;
-
-  private static String[] getPrefixes(String lex) {
-String[] prefs = new String[PREFIX_LENGTH];
-for (int li = 0; li < PREFIX_LENGTH; li++) {
-  prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
-}
-return prefs;
+  static final int DEFAULT_MAX_LENGTH = 4;
+  
+  private final int prefixLength;
+  
+  public PrefixFeatureGenerator() {
+prefixLength = DEFAULT_MAX_LENGTH;
+  }
+  
+  public PrefixFeatureGenerator(int prefixLength) {
+this.prefixLength = prefixLength;
   }
 
+  @Override
   public void createFeatures(List features, String[] tokens, int index,
   String[] previousOutcomes) {
-String[] prefs = PrefixFeatureGenerator.getPrefixes(tokens[index]);
+String[] prefs = getPrefixes(tokens[index]);
 for (String pref : prefs) {
   features.add("pre=" + pref);
 }
   }
+  
+  private String[] getPrefixes(String lex) {
+  
+int prefixes = Math.min(prefixLength, lex.length());
+
+String[] prefs = new String[prefixes];
+for (int li = 0; li < prefixes; li++) {
+  prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
+}
+return prefs;

[03/24] opennlp git commit: OPENNLP-983: Make suffix/prefix length configurable

2017-03-13 Thread colen 
OPENNLP-983: Make suffix/prefix length configurable

This closes #121


Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/b7d3abce
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/b7d3abce
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/b7d3abce

Branch: refs/heads/OPENNLP-778
Commit: b7d3abce569b5a4bf0ae39b24c6ac9920032db01
Parents: 929595d
Author: jzonthemtn 
Authored: Mon Feb 13 07:57:21 2017 -0500
Committer: Jörn Kottmann 
Committed: Thu Feb 16 10:04:08 2017 +0100

--
 .../tools/util/featuregen/GeneratorFactory.java | 22 -
 .../util/featuregen/PrefixFeatureGenerator.java | 32 +--
 .../util/featuregen/SuffixFeatureGenerator.java | 33 +--
 .../featuregen/PrefixFeatureGeneratorTest.java  | 92 
 .../featuregen/SuffixFeatureGeneratorTest.java  | 92 
 5 files changed, 251 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/opennlp/blob/b7d3abce/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
index fa97f43..ef08cfb 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
@@ -555,7 +555,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new PrefixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int prefixLength = PrefixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+prefixLength = Integer.parseInt(attribute);
+  }
+
+  return new PrefixFeatureGenerator(prefixLength);
 }
 
 static void register(Map factoryMap) {
@@ -570,7 +579,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new SuffixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int suffixLength = SuffixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+suffixLength = Integer.parseInt(attribute);
+  }
+
+  return new SuffixFeatureGenerator(suffixLength);
 }
 
 static void register(Map factoryMap) {

http://git-wip-us.apache.org/repos/asf/opennlp/blob/b7d3abce/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
index 8cdd48f..04fcd15 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
@@ -21,21 +21,35 @@ import java.util.List;
 
 public class PrefixFeatureGenerator implements AdaptiveFeatureGenerator {
 
-  private static final int PREFIX_LENGTH = 4;
-
-  private static String[] getPrefixes(String lex) {
-String[] prefs = new String[PREFIX_LENGTH];
-for (int li = 0; li < PREFIX_LENGTH; li++) {
-  prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
-}
-return prefs;
+  static final int DEFAULT_MAX_LENGTH = 4;
+  
+  private final int prefixLength;
+  
+  public PrefixFeatureGenerator() {
+prefixLength = DEFAULT_MAX_LENGTH;
+  }
+  
+  public PrefixFeatureGenerator(int prefixLength) {
+this.prefixLength = prefixLength;
   }
 
+  @Override
   public void createFeatures(List features, String[] tokens, int index,
   String[] previousOutcomes) {
-String[] prefs = PrefixFeatureGenerator.getPrefixes(tokens[index]);
+String[] prefs = getPrefixes(tokens[index]);
 for (String pref : prefs) {
   features.add("pre=" + pref);
 }
   }
+  
+  private String[] getPrefixes(String lex) {
+  
+int prefixes = Math.min(prefixLength, lex.length());
+
+String[] prefs = new String[prefixes];
+for (int li = 0; li < prefixes; li++) {
+  prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
+}
+return prefs;
+  }

opennlp git commit: OPENNLP-983: Make suffix/prefix length configurable

2017-02-16 Thread joern 
Repository: opennlp
Updated Branches:
  refs/heads/master 929595d2f -> b7d3abce5


OPENNLP-983: Make suffix/prefix length configurable

This closes #121


Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/b7d3abce
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/b7d3abce
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/b7d3abce

Branch: refs/heads/master
Commit: b7d3abce569b5a4bf0ae39b24c6ac9920032db01
Parents: 929595d
Author: jzonthemtn 
Authored: Mon Feb 13 07:57:21 2017 -0500
Committer: Jörn Kottmann 
Committed: Thu Feb 16 10:04:08 2017 +0100

--
 .../tools/util/featuregen/GeneratorFactory.java | 22 -
 .../util/featuregen/PrefixFeatureGenerator.java | 32 +--
 .../util/featuregen/SuffixFeatureGenerator.java | 33 +--
 .../featuregen/PrefixFeatureGeneratorTest.java  | 92 
 .../featuregen/SuffixFeatureGeneratorTest.java  | 92 
 5 files changed, 251 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/opennlp/blob/b7d3abce/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
index fa97f43..ef08cfb 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
@@ -555,7 +555,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new PrefixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int prefixLength = PrefixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+prefixLength = Integer.parseInt(attribute);
+  }
+
+  return new PrefixFeatureGenerator(prefixLength);
 }
 
 static void register(Map factoryMap) {
@@ -570,7 +579,16 @@ public class GeneratorFactory {
 
 public AdaptiveFeatureGenerator create(Element generatorElement,
 FeatureGeneratorResourceProvider resourceManager) {
-  return new SuffixFeatureGenerator();
+
+  String attribute = generatorElement.getAttribute("length");
+
+  int suffixLength = SuffixFeatureGenerator.DEFAULT_MAX_LENGTH;
+
+  if (!Objects.equals(attribute, "")) {
+suffixLength = Integer.parseInt(attribute);
+  }
+
+  return new SuffixFeatureGenerator(suffixLength);
 }
 
 static void register(Map factoryMap) {

http://git-wip-us.apache.org/repos/asf/opennlp/blob/b7d3abce/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
--
diff --git 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
index 8cdd48f..04fcd15 100644
--- 
a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
+++ 
b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
@@ -21,21 +21,35 @@ import java.util.List;
 
 public class PrefixFeatureGenerator implements AdaptiveFeatureGenerator {
 
-  private static final int PREFIX_LENGTH = 4;
-
-  private static String[] getPrefixes(String lex) {
-String[] prefs = new String[PREFIX_LENGTH];
-for (int li = 0; li < PREFIX_LENGTH; li++) {
-  prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
-}
-return prefs;
+  static final int DEFAULT_MAX_LENGTH = 4;
+  
+  private final int prefixLength;
+  
+  public PrefixFeatureGenerator() {
+prefixLength = DEFAULT_MAX_LENGTH;
+  }
+  
+  public PrefixFeatureGenerator(int prefixLength) {
+this.prefixLength = prefixLength;
   }
 
+  @Override
   public void createFeatures(List features, String[] tokens, int index,
   String[] previousOutcomes) {
-String[] prefs = PrefixFeatureGenerator.getPrefixes(tokens[index]);
+String[] prefs = getPrefixes(tokens[index]);
 for (String pref : prefs) {
   features.add("pre=" + pref);
 }
   }
+  
+  private String[] getPrefixes(String lex) {
+  
+int prefixes = Math.min(prefixLength, lex.length());
+
+String[] prefs = new String[prefixes];
+for (int li = 0; li < prefixes; li++) {
+  prefs[li] =