big bugfix: was detokenizing target side, instead of tokenizing
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/fa1ea3e8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/fa1ea3e8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/fa1ea3e8 Branch: refs/heads/master Commit: fa1ea3e85b36ffc22354322125ee07d8e0b4254f Parents: 21eae7d Author: Matt Post <[email protected]> Authored: Mon Mar 28 14:46:23 2016 -0400 Committer: Matt Post <[email protected]> Committed: Mon Mar 28 14:46:23 2016 -0400 ---------------------------------------------------------------------- scripts/training/pipeline.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/fa1ea3e8/scripts/training/pipeline.pl ---------------------------------------------------------------------- diff --git a/scripts/training/pipeline.pl b/scripts/training/pipeline.pl index f43ee8d..7b241b0 100755 --- a/scripts/training/pipeline.pl +++ b/scripts/training/pipeline.pl @@ -71,7 +71,7 @@ my $DO_SUBSAMPLE = 0; my $DO_PACK_GRAMMARS = 1; my $SCRIPTDIR = "$JOSHUA/scripts"; my $TOKENIZER_SOURCE = "$SCRIPTDIR/preparation/tokenize.pl"; -my $TOKENIZER_TARGET = "$SCRIPTDIR/preparation/detokenize.pl"; +my $TOKENIZER_TARGET = "$SCRIPTDIR/preparation/tokenize.pl"; my $NORMALIZER = "$SCRIPTDIR/preparation/normalize.pl"; my $LOWERCASER = "$SCRIPTDIR/preparation/lowercase.pl"; my $GIZA_TRAINER = "$SCRIPTDIR/training/run-giza.pl";
