Fixed UniqueHypothesesTest, SourceAnnotationsTest and OOVListTest to work with 
new configuration. Compiles but does not pass yet.


Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/3ed109af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/3ed109af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/3ed109af

Branch: refs/heads/7_confsystem
Commit: 3ed109afab7c91dc93a92a387053bc140029784b
Parents: 3b70a97
Author: Michael A. Hedderich <michael-al...@users.noreply.github.com>
Authored: Sun Sep 18 19:20:12 2016 +0200
Committer: Michael A. Hedderich <michael-al...@users.noreply.github.com>
Committed: Sun Sep 18 19:23:35 2016 +0200

----------------------------------------------------------------------
 .../apache/joshua/decoder/cky/OOVListTest.conf  | 28 ++++++++++++++++
 .../apache/joshua/decoder/cky/OOVListTest.gold  |  3 ++
 .../apache/joshua/decoder/cky/OOVListTest.in    |  3 ++
 .../apache/joshua/decoder/cky/OOVListTest.java  | 34 ++++++--------------
 .../cky/SourceAnnotationsNotUsingTest.conf      | 27 ++++++++++++++++
 .../decoder/cky/SourceAnnotationsTest.java      | 28 ++++++++--------
 .../decoder/cky/SourceAnnotationsUsingTest.conf | 28 ++++++++++++++++
 .../decoder/cky/UniqueHypothesesTest.config     | 32 ++++++++++++++++++
 .../decoder/cky/UniqueHypothesesTest.java       | 19 +++++------
 9 files changed, 152 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf
new file mode 100644
index 0000000..d045e22
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf
@@ -0,0 +1,28 @@
+mark_oovs = true
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+use_unique_nbest = true
+use_tree_nbest = false
+top_n = 1
+oov_list = CD 0.0488752 JJ 0.186114 NN 0.291795 NNS 0.0894967 NP 0.117171 OOV 
0.033015 VB 0.0313967 VBG 0.0404596 VBN 0.0317203
+output_format = %s ||| %f ||| %c
+
+feature_functions = [
+  { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = 
src/test/resources/decoder/oov-list/lm.gz }
+  { class = WordPenalty }
+  { class = OOVPenalty }
+]
+
+grammars = [
+  { class = TextGrammar, owner = phrase, span_limit = 20, path = 
src/test/resources/decoder/oov-list/grammar }
+  { class = TextGrammar, owner = glue, span_limit = -1, path = 
src/test/resources/decoder/oov-list/glue-grammar }
+]
+
+weights = {
+  OOVPenalty = 1.0
+  WordPenalty = -3.6942747832593694
+  glue_0 = -1
+  lm_0 = 1.2373676802179452
+  phrase_0 = -1
+}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold
new file mode 100644
index 0000000..7cebb7b
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold
@@ -0,0 +1,3 @@
+Goats eat cheese ||| phrase_0=0.000000 lm_0=-16.587435 glue_0=-1.000000 
WordPenalty=-2.171473 ||| -11.503
+i will go home ||| phrase_0=0.000000 lm_0=-12.155182 glue_0=-1.000000 
WordPenalty=-2.605767 ||| -4.414
+goets_OOV eet_OOV cheez_OOV ||| phrase_0=0.000000 lm_0=-17.699974 
glue_0=-1.000000 WordPenalty=-2.171473 OOVPenalty=-7.749294 ||| -20.629

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in
new file mode 100644
index 0000000..23fd45a
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in
@@ -0,0 +1,3 @@
+CHEESE GOATS EAT
+3 1 4 2
+goets eet cheez

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
index 86282bf..394d806 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
@@ -18,20 +18,17 @@
  */
 package org.apache.joshua.decoder.cky;
 
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static 
org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
 
 import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
 import org.apache.joshua.util.io.KenLmTestUtil;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.Test;
 
+import com.typesafe.config.Config;
+
 public class OOVListTest {
-  private JoshuaConfiguration joshuaConfig;
   private Decoder decoder;
 
   @AfterMethod
@@ -44,23 +41,12 @@ public class OOVListTest {
 
   @Test
   public void 
givenInput_whenDecodingWithOOVList_thenScoreAndTranslationCorrect() throws 
Exception {
-    // Given
-    List<String> inputStrings = loadStringsFromFile(
-        "src/test/resources/decoder/oov-list/input.txt");
-
-    // When
-    configureDecoder("src/test/resources/decoder/oov-list/joshua.config");
-    List<String> decodedStrings = decodeList(inputStrings, decoder, 
joshuaConfig);
-
-    // Then
-    List<String> goldStrings = loadStringsFromFile(
-        "src/test/resources/decoder/oov-list/output.gold");
-    assertEquals(decodedStrings, goldStrings);
-  }
+    String inputPath = this.getClass().getResource("OOVListTest.in").getFile();
+    String goldPath = 
this.getClass().getResource("OOVListTest.gold").getFile();
+    Config config = parseResources(this.getClass(), "OOVListTest.conf")
+        .withFallback(Decoder.getDefaultFlags());
+    KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
 
-  public void configureDecoder(String pathToConfig) throws Exception {
-    joshuaConfig = new JoshuaConfiguration();
-    joshuaConfig.readConfigFile(pathToConfig);
-    KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
+    decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
new file mode 100644
index 0000000..a37e9b0
--- /dev/null
+++ 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
@@ -0,0 +1,27 @@
+default_non_terminal = X
+goal_symbol = GOAL
+mark_oovs = true
+pop_limit = 100
+top_n = 1
+use_unique_nbest = true
+output_format = %s ||| %f ||| %c
+include_align_index = false
+feature_function = OOVPenalty
+feature_function = WordPenalty
+
+feature_functions = [
+  { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = 
src/test/resources/decoder/source-annotations/lm.kenlm }
+]
+
+grammars = [
+  { class = TextGrammar, owner = pt, span_limit = 20, path = 
src/test/resources/decoder/source-annotations/grammar }
+  { class = TextGrammar, owner = glue, span_limit = -1, path = 
src/test/resources/decoder/source-annotations/grammar.glue }
+]
+
+weights = {
+  OOVPenalty = 1
+  WordPenalty = -1.5244636836685694
+  glue_0 = -0.1663815584150378
+  lm_0 = 1.0
+  pt_0 = -0.049141264495762726
+}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
index 28fc9e3..ee0f7aa 100644
--- 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
+++ 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
@@ -18,45 +18,43 @@
  */
 package org.apache.joshua.decoder.cky;
 
+import static com.typesafe.config.ConfigFactory.parseResources;
 import static org.apache.joshua.decoder.cky.TestUtil.translate;
 import static org.testng.Assert.assertEquals;
 
 import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
 import org.apache.joshua.util.io.KenLmTestUtil;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.Test;
 
+import com.typesafe.config.Config;
+
 public class SourceAnnotationsTest {
 
   private static final String INPUT = "mis[tag=ADJ;num=PL;class=OOV] amigos me 
llaman";
   private static final String GOLD_WITHOUT_ANNOTATIONS = "my friends call me 
||| pt_0=3.000000 lm_0=-11.973694 glue_0=-3.000000 WordPenalty=-2.605767 ||| 
-7.650";
   private static final String GOLD_WITH_ANNOTATIONS = "my friends call me ||| 
pt_0=3.000000 lm_0=-111.512733 glue_0=-3.000000 WordPenalty=-2.605767 ||| 
-107.189";
 
-  private static final String JOSHUA_CONFIG_PATH = 
"src/test/resources/decoder/source-annotations/joshua.config";
-
-  private JoshuaConfiguration joshuaConfig;
   private Decoder decoder;
 
   @Test
   public void givenInput_whenNotUsingSourceAnnotations_thenOutputCorrect() 
throws Exception {
-    setUp(false);
-    String output = translate(INPUT, decoder, joshuaConfig);
+    Config config = parseResources(this.getClass(), 
"SourceAnnotationsNotUsingTest.conf")
+        .withFallback(Decoder.getDefaultFlags());
+    KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
+
+    String output = translate(INPUT, decoder);
     assertEquals(output.trim(), GOLD_WITHOUT_ANNOTATIONS);
   }
 
   @Test
   public void givenInput_whenUsingSourceAnnotations_thenOutputCorrect() throws 
Exception {
-    setUp(true);
-    String output = translate(INPUT, decoder, joshuaConfig);
-    assertEquals(output.trim(), GOLD_WITH_ANNOTATIONS);
-  }
+    Config config = parseResources(this.getClass(), 
"SourceAnnotationsUsingTest.conf")
+        .withFallback(Decoder.getDefaultFlags());
+    KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
 
-  public void setUp(boolean sourceAnnotations) throws Exception {
-    joshuaConfig = new JoshuaConfiguration();
-    joshuaConfig.readConfigFile(JOSHUA_CONFIG_PATH);
-    joshuaConfig.source_annotations = sourceAnnotations;
-    KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
+    String output = translate(INPUT, decoder);
+    assertEquals(output.trim(), GOLD_WITH_ANNOTATIONS);
   }
 
   @AfterMethod

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
new file mode 100644
index 0000000..7533ee0
--- /dev/null
+++ 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
@@ -0,0 +1,28 @@
+default_non_terminal = X
+goal_symbol = GOAL
+mark_oovs = true
+pop_limit = 100
+top_n = 1
+use_unique_nbest = true
+output_format = %s ||| %f --- %c
+include_align_index = false
+feature_function = OOVPenalty
+feature_function = WordPenalty
+source_annotations = true
+
+feature_functions = [
+  { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = 
src/test/resources/decoder/source-annotations/lm.kenlm }
+]
+
+grammars = [
+  { class = TextGrammar, owner = pt, span_limit = 20, path = 
src/test/resources/decoder/source-annotations/grammar }
+  { class = TextGrammar, owner = glue, span_limit = -1, path = 
src/test/resources/decoder/source-annotations/grammar.glue }
+]
+
+weights = {
+  OOVPenalty = 1
+  WordPenalty = -1.5244636836685694
+  glue_0 = -0.1663815584150378
+  lm_0 = 1.0
+  pt_0 = -0.049141264495762726
+}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
new file mode 100644
index 0000000..b7172ef
--- /dev/null
+++ 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
@@ -0,0 +1,32 @@
+default_non_terminal = X
+goal_symbol = GOAL
+mark_oovs = false
+pop_limit = 100
+top_n = 300
+use_unique_nbest = true
+output_format = %s
+include_align_index = false
+feature_function = Distortion
+feature_function = PhrasePenalty
+
+feature_functions = [
+  { class = StateMinimizingLanguageModel, lm_type = kenlm, lm_order = 5, 
lm_file = src/test/resources/decoder/phrase/unique-hypotheses/lm.1.gz }
+  { class = OOVPenalty }
+  { class = WordPenalty }
+]
+
+grammars = [
+  { class = TextGrammar, owner = pt, span_limit = 0, path = 
src/test/resources/decoder/phrase/unique-hypotheses/rules.1.gz }
+]
+
+weights = {
+  Distortion = 1.0
+  OOVPenalty = 1.0
+  PhrasePenalty = 1.0
+  WordPenalty = -2.844814
+  lm_0 = 1.0
+  tm_pt_0 = 1.0
+  tm_pt_1 = 1.0
+  tm_pt_2 = 1.0
+  tm_pt_3 = 1.0
+}

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
----------------------------------------------------------------------
diff --git 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
index 825de45..6e163e5 100644
--- 
a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
+++ 
b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.joshua.decoder.cky;
 
+import static com.typesafe.config.ConfigFactory.parseResources;
 import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
 import static org.testng.Assert.assertEquals;
 
@@ -27,11 +28,12 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
 import org.apache.joshua.util.io.KenLmTestUtil;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.Test;
 
+import com.typesafe.config.Config;
+
 /**
  * Ensures that derivations are unique for the phrase-based decoder.
  */
@@ -39,15 +41,16 @@ public class UniqueHypothesesTest {
 
   public static final String INPUT = "una estrategia republicana para 
obstaculizar la reelección de Obama";
 
-  private JoshuaConfiguration joshuaConfig = null;
   private Decoder decoder = null;
 
   @Test
   public void 
givenInputSentence_whenDecodingWithUniqueHypotheses_thenAllHypothesesUnique()
       throws Exception {
-    
configureDecoder("src/test/resources/decoder/phrase/unique-hypotheses/joshua.config");
-    List<String> decodedStrings = decodeList(Arrays.asList(new String[] { 
INPUT }), decoder,
-        joshuaConfig);
+    Config config = parseResources(this.getClass(), 
"UniqueHypothesesTest.config")
+        .withFallback(Decoder.getDefaultFlags());
+    KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
+
+    List<String> decodedStrings = decodeList(Arrays.asList(new String[] { 
INPUT }), decoder);
 
     assertEquals(decodedStrings.size(), 300);
 
@@ -57,12 +60,6 @@ public class UniqueHypothesesTest {
     assertEquals(decodedStrings.size(), uniqueDecodedStrings.size());
   }
 
-  public void configureDecoder(String pathToConfig) throws Exception {
-    joshuaConfig = new JoshuaConfiguration();
-    joshuaConfig.readConfigFile(pathToConfig);
-    KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
-  }
-
   @AfterMethod
   public void tearDown() throws Exception {
     if (decoder != null) {

Reply via email to