Fix: tuner utilize threads when NUM_THREADS is set

Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/97c155f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/97c155f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/97c155f4

Branch: refs/heads/master
Commit: 97c155f4f378504ffd59bfd87766002014ad714f
Parents: cad86aa
Author: Thamme Gowda <t...@isi.edu>
Authored: Mon Mar 5 17:53:09 2018 -0800
Committer: Thamme Gowda <t...@isi.edu>
Committed: Mon Mar 5 17:53:09 2018 -0800

----------------------------------------------------------------------
 scripts/training/pipeline.pl  | 1 +
 scripts/training/run_tuner.py | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/97c155f4/scripts/training/pipeline.pl
----------------------------------------------------------------------
diff --git a/scripts/training/pipeline.pl b/scripts/training/pipeline.pl
index b057dac..c5dc228 100755
--- a/scripts/training/pipeline.pl
+++ b/scripts/training/pipeline.pl
@@ -173,6 +173,7 @@ my $NUM_JOBS = 1;
 # The number of threads to use at different pieces in the pipeline
 # (giza, decoding)
 my $NUM_THREADS = 1;
+$ENV{'NUM_THREADS'} = $NUM_THREADS;
 
 # which LM to use (kenlm or berkeleylm)
 my $LM_TYPE = "kenlm";

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/97c155f4/scripts/training/run_tuner.py
----------------------------------------------------------------------
diff --git a/scripts/training/run_tuner.py b/scripts/training/run_tuner.py
index d548aee..6d1f69c 100755
--- a/scripts/training/run_tuner.py
+++ b/scripts/training/run_tuner.py
@@ -347,10 +347,11 @@ def parse_tm_line(line):
 def get_features(config_file):
     """Queries the decoder for all dense features that will be fired by the 
feature
     functions activated in the config file"""
-
-    mem_size = os.environ.get('JOSHUA_MEM', None)
-    mem_arg = '-m %s' % mem_size if mem_size else ''
+    
+    mem_arg = '-m %s' % os.environ['JOSHUA_MEM'] if 'JOSHUA_MEM' in os.environ 
else ''
     decode_cmd = "%s/bin/joshua-decoder %s -c %s -show-weights -v 0" % 
(JOSHUA, mem_arg, config_file)
+    if 'NUM_THREADS' in os.environ:
+        decode_cmd += ' -threads %s' % os.environ['NUM_THREADS']
     output = check_output(decode_cmd, shell=True)
     features = []
     for index, item in enumerate(output.split('\n'.encode(encoding='utf_8', 
errors='strict'))):

Reply via email to