Repository: systemml Updated Branches: refs/heads/master c27c488be -> 50b3c80c6
[MINOR] bug fix in perf test script Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/50b3c80c Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/50b3c80c Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/50b3c80c Branch: refs/heads/master Commit: 50b3c80c61b9b6ab6cdb8b28a1544952fb8dd19d Parents: c27c488 Author: Nakul Jindal <naku...@gmail.com> Authored: Mon Oct 2 17:54:17 2017 -0700 Committer: Nakul Jindal <naku...@gmail.com> Committed: Mon Oct 2 17:54:17 2017 -0700 ---------------------------------------------------------------------- scripts/perftest/python/datagen.py | 4 +-- scripts/perftest/python/predict.py | 4 +-- scripts/perftest/python/train.py | 4 +-- scripts/perftest/python/utils_exec.py | 41 ++++++++++++++++++++---------- scripts/perftest/python/utils_fs.py | 2 +- scripts/perftest/python/utils_misc.py | 4 +-- 6 files changed, 37 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/datagen.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/datagen.py b/scripts/perftest/python/datagen.py index 6794187..54f2eff 100755 --- a/scripts/perftest/python/datagen.py +++ b/scripts/perftest/python/datagen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#------------------------------------------------------------- +# ------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,7 +18,7 @@ # specific language governing permissions and limitations # under the License. # -#------------------------------------------------------------- +# ------------------------------------------------------------- import itertools from os.path import join http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/predict.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/predict.py b/scripts/perftest/python/predict.py index 67467b1..56ba999 100755 --- a/scripts/perftest/python/predict.py +++ b/scripts/perftest/python/predict.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#------------------------------------------------------------- +# ------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,7 +18,7 @@ # specific language governing permissions and limitations # under the License. # -#------------------------------------------------------------- +# ------------------------------------------------------------- import sys from os.path import join http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/train.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/train.py b/scripts/perftest/python/train.py index a95950d..4428e8f 100755 --- a/scripts/perftest/python/train.py +++ b/scripts/perftest/python/train.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#------------------------------------------------------------- +# ------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,7 +18,7 @@ # specific language governing permissions and limitations # under the License. # -#------------------------------------------------------------- +# ------------------------------------------------------------- import sys from os.path import join http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/utils_exec.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/utils_exec.py b/scripts/perftest/python/utils_exec.py index 27f75f2..d51cf2d 100755 --- a/scripts/perftest/python/utils_exec.py +++ b/scripts/perftest/python/utils_exec.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#------------------------------------------------------------- +# ------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,12 +18,14 @@ # specific language governing permissions and limitations # under the License. # -#------------------------------------------------------------- +# ------------------------------------------------------------- import sys import subprocess import shlex import re +import tempfile +import os # Subprocess and log parsing related functions @@ -49,24 +51,34 @@ def subprocess_exec(cmd_string, log_file_path=None, extract=None): # print(cmd_string) exec_command = shlex.split(cmd_string) - proc1 = subprocess.Popen(exec_command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + log_file = None + is_temp_file = False - error_arr, out_arr = get_all_logs(proc1) - std_outs = out_arr + error_arr - std_outs.insert(0, ' '.join(exec_command)) + if log_file_path is not None: + log_file_path = log_file_path + '.log' + log_file = open(log_file_path, "w+") + else: + os_log_file, log_file_path = tempfile.mkstemp() + log_file = os.fdopen(os_log_file, 'w+') + is_temp_file = True + + log_file.write(' '.join(exec_command)) + log_file.write('\n') + proc1 = subprocess.Popen(exec_command, stdout=log_file, + stderr=subprocess.STDOUT) + proc1.wait() return_code = proc1.returncode - if log_file_path is not None: - write_logs(std_outs, log_file_path + '.log') + log_file.close() + log_file = open(log_file_path, 'r+') if return_code == 0: if extract == 'time': - return_data = parse_time(std_outs) + return_data = parse_time(log_file) if extract == 'dir': - return_data = parse_hdfs_paths(std_outs) + return_data = parse_hdfs_paths(log_file) if extract == 'hdfs_base': - return_data = parse_hdfs_base(std_outs) + return_data = parse_hdfs_base(log_file) if extract is None: return_data = 0 @@ -74,9 +86,12 @@ def subprocess_exec(cmd_string, log_file_path=None, extract=None): return_data = 'proc_fail' print('sub-process failed, return code {}'.format(return_code)) - return return_data + if is_temp_file: + os.remove(log_file_path) + return return_data + def parse_hdfs_base(std_outs): """ return: String http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/utils_fs.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/utils_fs.py b/scripts/perftest/python/utils_fs.py index 0e3a7b6..a604ca1 100755 --- a/scripts/perftest/python/utils_fs.py +++ b/scripts/perftest/python/utils_fs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#------------------------------------------------------------- +# ------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/utils_misc.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/utils_misc.py b/scripts/perftest/python/utils_misc.py index 87b870b..ebc4376 100755 --- a/scripts/perftest/python/utils_misc.py +++ b/scripts/perftest/python/utils_misc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#------------------------------------------------------------- +# ------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,7 +18,7 @@ # specific language governing permissions and limitations # under the License. # -#------------------------------------------------------------- +# ------------------------------------------------------------- from os.path import join import os