AMBARI-18814 : Add common log rotation settings to HDFS configs (Madhuvanthi Radhakrishnan via avijayan).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2b91ffda Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2b91ffda Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2b91ffda Branch: refs/heads/branch-dev-patch-upgrade Commit: 2b91ffda82194db5aa09f164c490e2d9767dc643 Parents: 02f649f Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Tue Dec 20 15:54:58 2016 -0800 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Tue Dec 20 15:54:58 2016 -0800 ---------------------------------------------------------------------- .../HDFS/2.1.0.2.0/configuration/hdfs-log4j.xml | 50 ++++++++++++++++++-- .../2.0.6/hooks/before-START/scripts/params.py | 6 +++ .../scripts/shared_initialization.py | 2 +- .../services/HDFS/configuration/hdfs-log4j.xml | 50 ++++++++++++++++++-- .../hooks/before-START/test_before_start.py | 8 ++-- 5 files changed, 103 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2b91ffda/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-log4j.xml index 7e9db57..52f166d 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-log4j.xml @@ -20,6 +20,48 @@ */ --> <configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>hadoop_security_log_max_backup_size</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Hadoop Security Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hadoop_security_log_number_of_backup_files</name> + <value>20</value> + <description>The number of backup files</description> + <display-name>Hadoop Security Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hadoop_log_max_backup_size</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Hadoop Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hadoop_log_number_of_backup_files</name> + <value>20</value> + <description>The number of backup files</description> + <display-name>Hadoop Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> <property> <name>content</name> <display-name>hdfs-log4j template</display-name> @@ -112,8 +154,8 @@ log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n #Security audit appender # hadoop.security.logger=INFO,console -hadoop.security.log.maxfilesize=256MB -hadoop.security.log.maxbackupindex=20 +hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB +hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}} log4j.category.SecurityLogger=${hadoop.security.logger} hadoop.security.log.file=SecurityAuth.audit log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender @@ -161,8 +203,8 @@ log4j.appender.RFA=org.apache.log4j.RollingFileAppender log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} # Logfile size and and 30-day backups -log4j.appender.RFA.MaxFileSize=256MB -log4j.appender.RFA.MaxBackupIndex=10 +log4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB +log4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}} log4j.appender.RFA.layout=org.apache.log4j.PatternLayout log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n http://git-wip-us.apache.org/repos/asf/ambari/blob/2b91ffda/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index b905c2b..2ade512 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -208,6 +208,12 @@ yarn_log_dir_prefix = default("/configurations/yarn-env/yarn_log_dir_prefix","/v dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None) +# Hdfs log4j settings +hadoop_log_max_backup_size = default('configurations/hdfs-log4j/hadoop_log_max_backup_size', 256) +hadoop_log_number_of_backup_files = default('configurations/hdfs-log4j/hadoop_log_number_of_backup_files', 20) +hadoop_security_log_max_backup_size = default('configurations/hdfs-log4j/hadoop_security_log_max_backup_size', 256) +hadoop_security_log_number_of_backup_files = default('configurations/hdfs-log4j/hadoop_security_log_number_of_backup_files', 20) + # Yarn log4j settings yarn_rm_summary_log_max_backup_size = default('configurations/yarn-log4j/yarn_rm_summary_log_max_backup_size', 256) yarn_rm_summary_log_number_of_backup_files = default('configurations/yarn-log4j/yarn_rm_summary_log_number_of_backup_files', 20) http://git-wip-us.apache.org/repos/asf/ambari/blob/2b91ffda/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index 5dce8e0..cf958f0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -89,7 +89,7 @@ def setup_hadoop(): mode=0644, group=params.user_group, owner=params.hdfs_user, - content=params.log4j_props + content=InlineTemplate(params.log4j_props) ) elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))): File(log4j_filename, http://git-wip-us.apache.org/repos/asf/ambari/blob/2b91ffda/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml index 4bf4cfe..7bdb938 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml @@ -20,6 +20,48 @@ */ --> <configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>hadoop_security_log_max_backup_size</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Hadoop Security Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hadoop_security_log_number_of_backup_files</name> + <value>20</value> + <description>The number of backup files</description> + <display-name>Hadoop Security Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hadoop_log_max_backup_size</name> + <value>256</value> + <description>The maximum size of backup file before the log is rotated</description> + <display-name>Hadoop Log: backup file size</display-name> + <value-attributes> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hadoop_log_number_of_backup_files</name> + <value>20</value> + <description>The number of backup files</description> + <display-name>Hadoop Log: # of backup files</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> <property> <name>content</name> <display-name>hdfs-log4j template</display-name> @@ -112,8 +154,8 @@ log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n #Security audit appender # hadoop.security.logger=INFO,console -hadoop.security.log.maxfilesize=256MB -hadoop.security.log.maxbackupindex=20 +hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB +hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}} log4j.category.SecurityLogger=${hadoop.security.logger} hadoop.security.log.file=SecurityAuth.audit log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender @@ -175,8 +217,8 @@ log4j.appender.RFA=org.apache.log4j.RollingFileAppender log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} # Logfile size and and 30-day backups -log4j.appender.RFA.MaxFileSize=256MB -log4j.appender.RFA.MaxBackupIndex=10 +log4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB +log4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}} log4j.appender.RFA.layout=org.apache.log4j.PatternLayout log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n http://git-wip-us.apache.org/repos/asf/ambari/blob/2b91ffda/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py index 6e5561a..30561ac 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py @@ -69,7 +69,7 @@ class TestHookBeforeStart(RMFTestCase): mode=0644, group='hadoop', owner='hdfs', - content='log4jproperties\nline2log4jproperties\nline2' + content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2') ) self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties', content = InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']), @@ -143,7 +143,7 @@ class TestHookBeforeStart(RMFTestCase): mode=0644, group='hadoop', owner='hdfs', - content='log4jproperties\nline2log4jproperties\nline2' + content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2') ) self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties', content = InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']), @@ -222,7 +222,7 @@ class TestHookBeforeStart(RMFTestCase): mode=0644, group='hadoop', owner='hdfs', - content='log4jproperties\nline2log4jproperties\nline2' + content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2') ) self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties', content = InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']), @@ -303,7 +303,7 @@ class TestHookBeforeStart(RMFTestCase): mode=0644, group='hadoop', owner='hdfs', - content='log4jproperties\nline2log4jproperties\nline2' + content=InlineTemplate('log4jproperties\nline2log4jproperties\nline2') ) self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties', content = InlineTemplate(self.getConfig()['configurations']['hadoop-metrics2.properties']['content']),