AMBARI-20180. HDFS SecurityLogger messages are logged twice (Arpit Agarwal via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cb1ba69e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cb1ba69e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cb1ba69e Branch: refs/heads/branch-dev-logsearch Commit: cb1ba69e721326bfb4ef3531e8a9e509f36dbb4d Parents: ee6c5d8 Author: Sumit Mohanty <[email protected]> Authored: Tue Mar 7 16:52:55 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Tue Mar 7 16:53:57 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml | 1 + .../main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml | 6 ++++++ .../stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 4 ++++ .../src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 1 + .../main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 5 +++++ .../src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 1 + .../main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 5 +++++ .../src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + .../HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml | 1 + 11 files changed, 35 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml index 280c43a..9e48acc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml @@ -158,6 +158,7 @@ hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}} log4j.category.SecurityLogger=${hadoop.security.logger} hadoop.security.log.file=SecurityAuth.audit +log4j.additivity.SecurityLogger=false log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index 96a95fb..eeb10fc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -374,6 +374,12 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + + <definition xsi:type="configure" id="hdfs_securitylogger_additivity" summary="Set additivity of SecurityLogger to false"> + <type>hdfs-log4j</type> + <regex-replace key="content" find="hadoop.security.log.file=SecurityAuth.audit" replace-with="hadoop.security.log.file=SecurityAuth.audit log4j.additivity.SecurityLogger=false"/> + <regex-replace key="content" find="log4j.additivity.SecurityLogger=true" replace-with="log4j.additivity.SecurityLogger=false"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 40ef43d..c8f93e4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -293,6 +293,10 @@ </task> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hdfs NameNode"> + <task xsi:type="configure" id="hdfs_securitylogger_additivity"/> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index fb5e584..da5265d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -672,6 +672,7 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> + <task xsi:type="configure" id="hdfs_securitylogger_additivity" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 8ab88b8..1b809c5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -261,6 +261,11 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + <definition xsi:type="configure" id="hdfs_securitylogger_additivity" summary="Set additivity of SecurityLogger to false"> + <type>hdfs-log4j</type> + <regex-replace key="content" find="hadoop.security.log.file=SecurityAuth.audit" replace-with="hadoop.security.log.file=SecurityAuth.audit log4j.additivity.SecurityLogger=false"/> + <regex-replace key="content" find="log4j.additivity.SecurityLogger=true" replace-with="log4j.additivity.SecurityLogger=false"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index d5a7c68..53caa52 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -265,6 +265,11 @@ <direction>UPGRADE</direction> <!-- prevent config changes on downgrade --> <skippable>true</skippable> <!-- May fix configuration problems manually --> + <!--HDFS--> + <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hdfs NameNode"> + <task xsi:type="configure" id="hdfs_securitylogger_additivity"/> + </execute-stage> + <!--YARN--> <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index e11ff02..2ee2094 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -677,6 +677,7 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> + <task xsi:type="configure" id="hdfs_securitylogger_additivity" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 6f3fe42..febd8e2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -224,6 +224,11 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + <definition xsi:type="configure" id="hdfs_securitylogger_additivity" summary="Set additivity of SecurityLogger to false"> + <type>hdfs-log4j</type> + <regex-replace key="content" find="hadoop.security.log.file=SecurityAuth.audit" replace-with="hadoop.security.log.file=SecurityAuth.audit log4j.additivity.SecurityLogger=false"/> + <regex-replace key="content" find="log4j.additivity.SecurityLogger=true" replace-with="log4j.additivity.SecurityLogger=false"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index fe297c3..0d6f51b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -276,6 +276,11 @@ <direction>UPGRADE</direction> <!-- prevent config changes on downgrade --> <skippable>true</skippable> <!-- May fix configuration problems manually --> + <!--HDFS--> + <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hdfs NameNode"> + <task xsi:type="configure" id="hdfs_securitylogger_additivity"/> + </execute-stage> + <!--YARN--> <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 97313b5..ab6b239 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -605,6 +605,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> + <task xsi:type="configure" id="hdfs_securitylogger_additivity" /> </pre-upgrade> <pre-downgrade /> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb1ba69e/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml index 777c846..4d0cd73 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml @@ -115,6 +115,7 @@ hadoop.security.log.maxfilesize=256MB hadoop.security.log.maxbackupindex=20 log4j.category.SecurityLogger=${hadoop.security.logger} hadoop.security.log.file=SecurityAuth.audit +log4j.additivity.SecurityLogger=false log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
