Repository: systemml Updated Branches: refs/heads/master 95de23586 -> dc4bfd95e
[HOTFIX] write stdout and stderr for perftests Closes #615 Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/dc4bfd95 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/dc4bfd95 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/dc4bfd95 Branch: refs/heads/master Commit: dc4bfd95e893f924d80ca3af25c101495134fa77 Parents: 95de235 Author: krishnakalyan3 <krishnakaly...@gmail.com> Authored: Sat Aug 12 12:31:26 2017 -0700 Committer: Nakul Jindal <naku...@gmail.com> Committed: Sat Aug 12 12:32:16 2017 -0700 ---------------------------------------------------------------------- scripts/perftest/python/run_perftest.py | 2 +- scripts/perftest/python/utils_exec.py | 19 +++++++++++++++++-- scripts/perftest/python/utils_misc.py | 10 +++++----- 3 files changed, 23 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/dc4bfd95/scripts/perftest/python/run_perftest.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/run_perftest.py b/scripts/perftest/python/run_perftest.py index a15d7e6..d430569 100755 --- a/scripts/perftest/python/run_perftest.py +++ b/scripts/perftest/python/run_perftest.py @@ -134,7 +134,7 @@ def algorithm_workflow(algo, exec_type, config_path, dml_file_name, action_mode, if exit_flag_success: time = 'data_exists' else: - time = exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict) + time = exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict, config_path) write_success(time, temp_cwd) print('{},{},{},{},{},{}'.format(algo, action_mode, intercept, mat_type, mat_shape, time)) http://git-wip-us.apache.org/repos/asf/systemml/blob/dc4bfd95/scripts/perftest/python/utils_exec.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/utils_exec.py b/scripts/perftest/python/utils_exec.py index 0eb2873..87ae3cd 100755 --- a/scripts/perftest/python/utils_exec.py +++ b/scripts/perftest/python/utils_exec.py @@ -27,7 +27,7 @@ import re # Subprocess and log parsing related functions -def subprocess_exec(cmd_string, extract=None): +def subprocess_exec(cmd_string, log_file_path=None, extract=None): """ Execute the input string as subprocess @@ -38,18 +38,25 @@ def subprocess_exec(cmd_string, extract=None): Based on extract as time/dir we extract this information from the logs accordingly + log_file_path: String + Path to write the log file + return: String Based on extract we return the relevant string """ # Debug # print(cmd_string) - proc1 = subprocess.Popen(shlex.split(cmd_string), stdout=subprocess.PIPE, + exec_command = shlex.split(cmd_string) + proc1 = subprocess.Popen(exec_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) error_arr, out_arr = get_all_logs(proc1) std_outs = out_arr + error_arr return_code = proc1.returncode + if log_file_path is not None: + write_logs(std_outs, log_file_path + '.log') + if return_code == 0: if extract == 'time': return_data = parse_time(std_outs) @@ -65,6 +72,14 @@ def subprocess_exec(cmd_string, extract=None): return return_data +def write_logs(std_outs, log_file_path): + """ + Write all logs to the specified location + """ + with open(log_file_path, 'w')as log: + log.write("\n".join(std_outs)) + + def get_all_logs(process): """ Based on the subprocess capture logs http://git-wip-us.apache.org/repos/asf/systemml/blob/dc4bfd95/scripts/perftest/python/utils_misc.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/utils_misc.py b/scripts/perftest/python/utils_misc.py index 0a765f6..e247ce8 100755 --- a/scripts/perftest/python/utils_misc.py +++ b/scripts/perftest/python/utils_misc.py @@ -166,7 +166,7 @@ def config_reader(read_path): return conf_file -def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict): +def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict, log_file_name=None): """ This function is responsible of execution of input arguments via python sub process, We also extract time obtained from the output of this subprocess @@ -186,6 +186,9 @@ def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup sup_args_dict: Dictionary Supplementary arguments required by the script + log_file_name: String + Path to write the logfile + return: String The value of time parsed from the logs / error """ @@ -207,10 +210,7 @@ def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup cmd = [exec_script, spark_pre_args, '-f', algorithm, args, sup_args] cmd_string = ' '.join(cmd) - # Debug - # print(cmd_string) - - time = subprocess_exec(cmd_string, 'time') + time = subprocess_exec(cmd_string, log_file_name, 'time') return time
