Repository: ambari Updated Branches: refs/heads/trunk 17ead15a7 -> 8449cd1d3
AMBARI-5112 hadoop-mapreduce.jobsummary.log is empty when specified custom YARN Log Dir (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8449cd1d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8449cd1d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8449cd1d Branch: refs/heads/trunk Commit: 8449cd1d34929c823a90001e7e8d22590c2052e9 Parents: 17ead15 Author: Dmitry Sen <[email protected]> Authored: Wed Mar 19 17:23:55 2014 +0200 Committer: Dmitry Sen <[email protected]> Committed: Wed Mar 19 17:23:55 2014 +0200 ---------------------------------------------------------------------- .../MAPREDUCE/configuration/mapreduce-log4j.xml | 4 +++- .../services/YARN/configuration/yarn-log4j.xml | 4 +++- .../YARN/package/scripts/resourcemanager.py | 24 +------------------- .../2.0.6/services/YARN/package/scripts/yarn.py | 10 +++----- .../stacks/2.0.6/YARN/test_historyserver.py | 16 ------------- .../stacks/2.0.6/YARN/test_mapreduce2_client.py | 16 ------------- .../stacks/2.0.6/YARN/test_nodemanager.py | 16 ------------- .../stacks/2.0.6/YARN/test_resourcemanager.py | 8 ------- .../stacks/2.0.6/YARN/test_yarn_client.py | 24 -------------------- .../stacks/2.1/YARN/test_apptimelineserver.py | 8 ------- 10 files changed, 10 insertions(+), 120 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml index 529e063..c76c088 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml @@ -25,6 +25,8 @@ <property> <name>content</name> <value> +#Relative to Mapred LOG DIR Prefix +mapred.log.dir=. # # Job Summary Appender # @@ -36,7 +38,7 @@ hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.JSA.File={{hdfs_log_dir_prefix}}/{{mapred_user}}/${hadoop.mapreduce.jobsummary.log.file} +log4j.appender.JSA.File=${mapred.log.dir}/${hadoop.mapreduce.jobsummary.log.file} log4j.appender.JSA.layout=org.apache.log4j.PatternLayout log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml index 99501de..e04973e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml @@ -25,6 +25,8 @@ <property> <name>content</name> <value> +#Relative to Yarn Log Dir Prefix +yarn.log.dir=. # # Job Summary Appender # @@ -51,7 +53,7 @@ yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender -log4j.appender.RMSUMMARY.File={{yarn_log_dir_prefix}}/{{yarn_user}}/${yarn.server.resourcemanager.appsummary.log.file} +log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} log4j.appender.RMSUMMARY.MaxFileSize=256MB log4j.appender.RMSUMMARY.MaxBackupIndex=20 log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py index 4af82c7..6e6b34f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py @@ -20,28 +20,6 @@ Ambari Agent """ -#!/usr/bin/env python2.6 -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - import sys from resource_management import * @@ -58,7 +36,7 @@ class Resourcemanager(Script): import params env.set_params(params) - yarn() + yarn(name='resourcemanager') def start(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py index aa7f139..9387022 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py @@ -116,21 +116,17 @@ def yarn(name = None): mode=0644 ) - File(params.yarn_job_summary_log, + if name == 'resourcemanager': + File(params.yarn_job_summary_log, owner=params.yarn_user, group=params.user_group - ) + ) File(params.rm_nodes_exclude_path, owner=params.yarn_user, group=params.user_group ) - File(params.mapred_job_summary_log, - owner=params.mapred_user, - group=params.user_group - ) - File(format("{limits_conf_dir}/yarn.conf"), mode=0644, content=Template('yarn.conf.j2') http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py index 5c924f8..369e686 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py @@ -238,18 +238,10 @@ class TestHistoryServer(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, @@ -386,18 +378,10 @@ class TestHistoryServer(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py index e376850..e69dcfc 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py @@ -97,18 +97,10 @@ class TestMapReduce2Client(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, @@ -192,18 +184,10 @@ class TestMapReduce2Client(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py index 8e24f7a..8f694fc 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py @@ -238,18 +238,10 @@ class TestNodeManager(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, @@ -386,18 +378,10 @@ class TestNodeManager(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index ca870c0..f8cd4d3 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -186,10 +186,6 @@ class TestResourceManager(RMFTestCase): owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, @@ -275,10 +271,6 @@ class TestResourceManager(RMFTestCase): owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py index f57f055..0b15967 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py @@ -98,18 +98,10 @@ class TestYarnClient(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, @@ -193,18 +185,10 @@ class TestYarnClient(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, @@ -306,18 +290,10 @@ class TestYarnClient(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py index 233c57d..2046f52 100644 --- a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py +++ b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py @@ -132,18 +132,10 @@ class TestAppTimelineServer(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', ) - self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log', - owner = 'mapred', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644,
