Updated Branches: refs/heads/trunk 47e3d7ac2 -> a879a8310
AMBARI-3158. Ambari Hadoop: Job Summary Log should be configured to log as separate file. (Dmytro Shkvyra via odiachenko) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/a879a831 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/a879a831 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/a879a831 Branch: refs/heads/trunk Commit: a879a831080b6ce21bc56bec4beb013d7c34ce2f Parents: 47e3d7a Author: Oleksandr Diachenko <[email protected]> Authored: Tue Sep 24 20:28:26 2013 +0300 Committer: Oleksandr Diachenko <[email protected]> Committed: Tue Sep 24 20:28:49 2013 +0300 ---------------------------------------------------------------------- .../hdp-hadoop/templates/log4j.properties.erb | 17 ++++++++++++++--- .../puppet/modules/hdp-yarn/manifests/service.pp | 13 ++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/a879a831/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/log4j.properties.erb ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/log4j.properties.erb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/log4j.properties.erb index bc8d8b7..8e73d34 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/log4j.properties.erb +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/log4j.properties.erb @@ -162,6 +162,7 @@ log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender # log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter +<% if (scope.function_hdp_template_var("::hdp::params::is_jtnode_master") || scope.function_hdp_template_var("::hdp::params::is_rmnode_master"))%> # # Job Summary Appender # @@ -169,16 +170,26 @@ log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter # hadoop.mapreduce.jobsummary.log.file rolled daily: # hadoop.mapreduce.jobsummary.logger=INFO,JSA # -hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} +hadoop.mapreduce.jobsummary.logger=<%=scope.function_hdp_default(["mapred-site/mapreduce.reduce.log.level","INFO"])%>,JSA hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} +<% if (scope.function_hdp_get_major_stack_version([scope.function_hdp_template_var("stack_version")]) >= 2)%> +log4j.appender.JSA.File=<%=scope.function_hdp_template_var("yarn_log_dir_prefix")%>/<%=scope.function_hdp_template_var("yarn_user")%>/${hadoop.mapreduce.jobsummary.log.file} +<% else %> +log4j.appender.JSA.File=<%=scope.function_hdp_template_var("hdfs_log_dir_prefix")%>/<%=scope.function_hdp_template_var("mapred_user")%>/${hadoop.mapreduce.jobsummary.log.file} +<%end-%> log4j.appender.JSA.layout=org.apache.log4j.PatternLayout log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n log4j.appender.JSA.DatePattern=.yyyy-MM-dd +log4j.appender.JSA.layout=org.apache.log4j.PatternLayout +<% if (scope.function_hdp_get_major_stack_version([scope.function_hdp_template_var("stack_version")]) >= 2)%> +log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${hadoop.mapreduce.jobsummary.logger} +log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false +<% else %> log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false - +<%end-%> +<%end-%> <%=scope.function_hdp_template_var("rca_prefix")%>ambari.jobhistory.database=<%=scope.function_hdp_host("ambari_db_rca_url")%> <%=scope.function_hdp_template_var("rca_prefix")%>ambari.jobhistory.driver=<%=scope.function_hdp_host("ambari_db_rca_driver")%> http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/a879a831/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/service.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/service.pp b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/service.pp index 8b73016..3589829 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/service.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/service.pp @@ -30,15 +30,17 @@ define hdp-yarn::service( $security_enabled = $hdp::params::security_enabled if ($name == 'historyserver') { - $log_dir = "${hdp-yarn::params::mapred_log_dir_prefix}" + $log_dir = "${hdp-yarn::params::mapred_log_dir_prefix}/${user}" $pid_dir = "${hdp-yarn::params::mapred_pid_dir_prefix}/${user}" $daemon = "${hdp::params::mapred_bin}/mr-jobhistory-daemon.sh" $pid_file = "${pid_dir}/mapred-${user}-${name}.pid" + $job_summary_log = "${hdp-yarn::params::mapred_log_dir_prefix}/${user}/hadoop-mapreduce.jobsummary.log" } else { - $log_dir = "${hdp-yarn::params::yarn_log_dir_prefix}" + $log_dir = "${hdp-yarn::params::yarn_log_dir_prefix}/${user}" $pid_dir = "${hdp-yarn::params::yarn_pid_dir_prefix}/${user}" $daemon = "${hdp::params::yarn_bin}/yarn-daemon.sh" $pid_file = "${pid_dir}/yarn-${user}-${name}.pid" + $job_summary_log = "${hdp-yarn::params::yarn_log_dir_prefix}/${user}/hadoop-mapreduce.jobsummary.log" } $hadoop_libexec_dir = $hdp-yarn::params::hadoop_libexec_dir @@ -79,6 +81,11 @@ define hdp-yarn::service( service_state => $ensure, force => true } + + file {$job_summary_log: + path => $job_summary_log, + owner => $user, + } } if ($daemon_cmd != undef) { @@ -92,7 +99,7 @@ define hdp-yarn::service( anchor{"hdp-yarn::service::${name}::begin":} anchor{"hdp-yarn::service::${name}::end":} if ($daemon_cmd != undef) { - Anchor["hdp-yarn::service::${name}::begin"] -> Hdp::Directory_recursive_create<|title == $pid_dir or title == $log_dir|> -> Hdp::Exec[$daemon_cmd] -> Anchor["hdp-yarn::service::${name}::end"] + Anchor["hdp-yarn::service::${name}::begin"] -> Hdp::Directory_recursive_create<|title == $pid_dir or title == $log_dir|> -> File[$job_summary_log] -> Hdp::Exec[$daemon_cmd] -> Anchor["hdp-yarn::service::${name}::end"] } if ($ensure == 'running') {
