Repository: incubator-slider Updated Branches: refs/heads/develop 14ccea135 -> 4bebbf711
SLIDER-999 stdout and stderr spit out by application process on the agent side should be saved into a log file Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4bebbf71 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4bebbf71 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4bebbf71 Branch: refs/heads/develop Commit: 4bebbf7110d342c3b33fd0f5dba4146ce6a37ea1 Parents: 14ccea1 Author: Yu Liu <yu...@apache.org> Authored: Sun Mar 20 19:58:26 2016 -0700 Committer: Yu Liu <yu...@apache.org> Committed: Sun Mar 20 19:58:26 2016 -0700 ---------------------------------------------------------------------- slider-agent/src/main/python/agent/Constants.py | 4 +++ .../src/main/python/agent/DockerManager.py | 27 ++++++++++++++------ .../core/providers/windows/system.py | 15 ++++++++++- .../python/resource_management/core/shell.py | 22 +++++++++++++--- 4 files changed, 56 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4bebbf71/slider-agent/src/main/python/agent/Constants.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py index f120b94..6ede66e 100644 --- a/slider-agent/src/main/python/agent/Constants.py +++ b/slider-agent/src/main/python/agent/Constants.py @@ -34,3 +34,7 @@ ZK_QUORUM="zk_quorum" ZK_REG_PATH="zk_reg_path" AUTO_GENERATED="auto_generated" MAX_AM_CONNECT_RETRIES = 10 +APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application_" +APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = ".out" +APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application_" +APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = ".err" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4bebbf71/slider-agent/src/main/python/agent/DockerManager.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/DockerManager.py b/slider-agent/src/main/python/agent/DockerManager.py index d4da40a..023a163 100644 --- a/slider-agent/src/main/python/agent/DockerManager.py +++ b/slider-agent/src/main/python/agent/DockerManager.py @@ -48,8 +48,8 @@ class DockerManager(): if status_command_str: self.stored_status_command = status_command_str.split(" ") logger.info("status command" + str(self.stored_status_command)) - if command['hostLevelParams']: - if command['hostLevelParams']['container_id']: + if 'hostLevelParams' in command: + if 'container_id' in command['hostLevelParams']: self.container_id = command['hostLevelParams']['container_id'] if command['roleCommand'] == 'INSTALL': @@ -81,13 +81,17 @@ class DockerManager(): # will evolve into a class hierarch, linux and windows - def execute_command_on_linux(self, docker_command): + def execute_command_on_linux(self, docker_command, stdoutFile=None, stderrFile=None): command_str = '' for itr in docker_command: - command_str = command_str + ' ' + itr + command_str = command_str + ' ' + itr logger.info("command str: " + command_str) - proc = subprocess.Popen(command_str, stdout = subprocess.PIPE, shell=True) + if stdoutFile != None or stderrFile != None: + proc = subprocess.Popen(command_str, stdout = stdoutFile, + stderr = stderrFile, universal_newlines = True, shell=True) + else: + proc = subprocess.Popen(command_str, stdout = subprocess.PIPE, shell=True) out, err = proc.communicate() logger.info("docker command output: " + str(out) + " err: " + str(err)) return proc.returncode, out, err @@ -107,8 +111,6 @@ class DockerManager(): docker_command = [command_path, "run"] - #docker_command.append("--net=host") - if options: docker_command = self.add_docker_run_options_to_command(docker_command, options) if containerPort: @@ -124,8 +126,17 @@ class DockerManager(): docker_command.append(imageName) if additional_param: docker_command = self.add_additional_param_to_command(docker_command, additional_param) + #adding redirecting stdout stderr to file logger.info("docker run command: " + str(docker_command)) - return self.execute_command_on_linux(docker_command) + outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ + self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + + errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ + self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w') + stderrFile = open(errfilename, 'w') + return self.execute_command_on_linux(docker_command, stdoutFile, stderrFile) def add_docker_run_options_to_command(self, docker_command, options): return docker_command + options.split(" ") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4bebbf71/slider-agent/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py index daaa0cf..af513a1 100644 --- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py +++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py @@ -30,6 +30,10 @@ import subprocess import shutil from resource_management.libraries.script import Script +APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log' +APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err' def _merge_env(env1, env2, merge_keys=['PYTHONPATH']): """ @@ -69,7 +73,16 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= pid_file_name=None, poll_after=None): # TODO implement user Logger.info("Executing %s" % (command)) - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + #adding redirecting stdout stderr to file + outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ + str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + + errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ + str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w') + stderrFile = open(errfilename, 'w') + proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True, cwd=cwd, env=env, shell=False) code = None logAnyway = False http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4bebbf71/slider-agent/src/main/python/resource_management/core/shell.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py index 95d18fc..f21dbbf 100644 --- a/slider-agent/src/main/python/resource_management/core/shell.py +++ b/slider-agent/src/main/python/resource_management/core/shell.py @@ -31,6 +31,11 @@ from exceptions import ExecuteTimeoutException from resource_management.core.logger import Logger import time +APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log' +APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err' + def checked_call(command, logoutput=False, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None, poll_after=None): return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file, poll_after) @@ -63,10 +68,20 @@ def _call(command, logoutput=False, throw_on_failure=True, command = ["/bin/bash","--login","-c", command] """ command = ["/bin/bash","--login","-c", command] - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + #adding redirecting stdout stderr to file + outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ + str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + + errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ + str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w') + stderrFile = open(errfilename, 'w') + + proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) - + logAnyway = False if not wait_for_finish: if pid_file_name: @@ -91,7 +106,8 @@ def _call(command, logoutput=False, throw_on_failure=True, t = threading.Timer( timeout, on_timeout, [proc, q] ) t.start() - out = proc.communicate()[0].strip('\n') + #out = proc.communicate()[0].strip('\n') + out = proc.communicate() if timeout: if q.empty():
