SLIDER-1115 stdout/stderr files have errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66257d77 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66257d77 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66257d77 Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 66257d771055a96ba0a4d403f41dac4b3da994cf Parents: e6999d0 Author: Yu Liu <yu...@apache.org> Authored: Wed Jun 1 11:03:17 2016 -0700 Committer: Yu Liu <yu...@apache.org> Committed: Wed Jun 1 11:03:17 2016 -0700 ---------------------------------------------------------------------- slider-agent/src/main/python/agent/ActionQueue.py | 4 ++-- slider-agent/src/main/python/agent/Constants.py | 4 ++-- .../src/main/python/agent/DockerManager.py | 16 +++++++--------- .../src/main/python/agent/YarnDockerManager.py | 17 ++++++++--------- .../core/providers/windows/system.py | 14 ++++++-------- .../main/python/resource_management/core/shell.py | 17 +++++++---------- 6 files changed, 32 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py index 107d6c3..be6af53 100644 --- a/slider-agent/src/main/python/agent/ActionQueue.py +++ b/slider-agent/src/main/python/agent/ActionQueue.py @@ -72,8 +72,8 @@ class ActionQueue(threading.Thread): self.customServiceOrchestrator = CustomServiceOrchestrator(config, controller, self.queueOutAgentToggleLogger) - self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator) - self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator) + self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator) + self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator) def stop(self): http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/Constants.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py index 6ede66e..524db6a 100644 --- a/slider-agent/src/main/python/agent/Constants.py +++ b/slider-agent/src/main/python/agent/Constants.py @@ -34,7 +34,7 @@ ZK_QUORUM="zk_quorum" ZK_REG_PATH="zk_reg_path" AUTO_GENERATED="auto_generated" MAX_AM_CONNECT_RETRIES = 10 -APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application_" +APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application" APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = ".out" -APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application_" +APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application" APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = ".err" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/DockerManager.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/DockerManager.py b/slider-agent/src/main/python/agent/DockerManager.py index 023a163..48085fe 100644 --- a/slider-agent/src/main/python/agent/DockerManager.py +++ b/slider-agent/src/main/python/agent/DockerManager.py @@ -31,9 +31,10 @@ class DockerManager(): stored_command = '' container_id = '' - def __init__(self, tmpdir, workroot, customServiceOrchestrator): + def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator): self.tmpdir = tmpdir self.workroot = workroot + self.logDir = logDir self.customServiceOrchestrator = customServiceOrchestrator def execute_command(self, command, store_command=False): @@ -128,14 +129,11 @@ class DockerManager(): docker_command = self.add_additional_param_to_command(docker_command, additional_param) #adding redirecting stdout stderr to file logger.info("docker run command: " + str(docker_command)) - outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ - self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE - - errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ - self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE - - stdoutFile = open(outfilename, 'w') - stderrFile = open(errfilename, 'w') + outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w+') + stderrFile = open(errfilename, 'w+') return self.execute_command_on_linux(docker_command, stdoutFile, stderrFile) def add_docker_run_options_to_command(self, docker_command, options): http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/YarnDockerManager.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py index abdb2fb..4a4d6de 100644 --- a/slider-agent/src/main/python/agent/YarnDockerManager.py +++ b/slider-agent/src/main/python/agent/YarnDockerManager.py @@ -24,6 +24,7 @@ import subprocess import Constants import time import traceback +from AgentConfig import AgentConfig from resource_management import * logger = logging.getLogger() @@ -33,9 +34,10 @@ class YarnDockerManager(Script): stored_command = '' container_id = '' - def __init__(self, tmpdir, workroot, customServiceOrchestrator): + def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator): self.tmpdir = tmpdir self.workroot = workroot + self.logDir = logDir self.customServiceOrchestrator = customServiceOrchestrator def execute_command(self, command, store_command=False): @@ -168,14 +170,11 @@ class YarnDockerManager(Script): #extracting param needed by docker run from the command passed from AM startCommand = self.extract_config_from_command(command, 'docker.startCommand') #adding redirecting stdout stderr to file - outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ - self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE - - errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ - self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE - - stdoutFile = open(outfilename, 'w') - stderrFile = open(errfilename, 'w') + outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w+') + stderrFile = open(errfilename, 'w+') returncode,out,err = self.execute_command_on_linux(startCommand, False, stdoutFile, stderrFile) return returncode,out,err http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py index b85c020..a3d9126 100644 --- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py +++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py @@ -30,9 +30,9 @@ import subprocess import shutil from resource_management.libraries.script import Script -APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application' APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log' -APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application' APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err' def _merge_env(env1, env2, merge_keys=['PYTHONPATH']): @@ -74,14 +74,12 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= # TODO implement user Logger.info("Executing %s" % (command)) #adding redirecting stdout stderr to file - outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ - str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE - errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ - str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE - stdoutFile = open(outfilename, 'w') - stderrFile = open(errfilename, 'w') + stdoutFile = open(outfilename, 'w+') + stderrFile = open(errfilename, 'w+') proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True, cwd=cwd, env=env, shell=False) code = None http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/resource_management/core/shell.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py index f21dbbf..4a383c7 100644 --- a/slider-agent/src/main/python/resource_management/core/shell.py +++ b/slider-agent/src/main/python/resource_management/core/shell.py @@ -31,9 +31,9 @@ from exceptions import ExecuteTimeoutException from resource_management.core.logger import Logger import time -APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application' APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log' -APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-' +APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application' APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err' def checked_call(command, logoutput=False, @@ -69,14 +69,11 @@ def _call(command, logoutput=False, throw_on_failure=True, """ command = ["/bin/bash","--login","-c", command] #adding redirecting stdout stderr to file - outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \ - str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE - - errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \ - str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE - - stdoutFile = open(outfilename, 'w') - stderrFile = open(errfilename, 'w') + outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE + errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE + + stdoutFile = open(outfilename, 'w+') + stderrFile = open(errfilename, 'w+') proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True, cwd=cwd, env=env, shell=False,
