Repository: ambari Updated Branches: refs/heads/trunk 4b884e30b -> a5b7e900b
AMBARI-20180. HDFS SecurityLogger messages are logged twice (Arpit Agarwal via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a5b7e900 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a5b7e900 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a5b7e900 Branch: refs/heads/trunk Commit: a5b7e900b2ad02588a0be79e3134dd277f21406a Parents: 4b884e3 Author: Sumit Mohanty <[email protected]> Authored: Tue Mar 7 16:52:55 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Tue Mar 7 16:52:55 2017 -0800 ---------------------------------------------------------------------- .../HDFS/3.0.0.3.0/configuration/hdfs-log4j.xml | 1 + .../stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml | 1 + .../main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml | 6 ++++++ .../stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 4 ++++ .../src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 1 + .../main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 5 +++++ .../src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 1 + .../main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 +++++ .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 5 +++++ .../src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 + .../HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml | 1 + 12 files changed, 36 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-log4j.xml index 448f224..5f6ec3f 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-log4j.xml @@ -117,6 +117,7 @@ hadoop.security.log.maxbackupindex=20 log4j.category.SecurityLogger=${hadoop.security.logger} hadoop.security.log.file=SecurityAuth.audit + log4j.additivity.SecurityLogger=false log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml index 280c43a..9e48acc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hdfs-log4j.xml @@ -158,6 +158,7 @@ hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}} log4j.category.SecurityLogger=${hadoop.security.logger} hadoop.security.log.file=SecurityAuth.audit +log4j.additivity.SecurityLogger=false log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index 319de86..5f135e9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -374,6 +374,12 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + + <definition xsi:type="configure" id="hdfs_securitylogger_additivity" summary="Set additivity of SecurityLogger to false"> + <type>hdfs-log4j</type> + <regex-replace key="content" find="hadoop.security.log.file=SecurityAuth.audit" replace-with="hadoop.security.log.file=SecurityAuth.audit log4j.additivity.SecurityLogger=false"/> + <regex-replace key="content" find="log4j.additivity.SecurityLogger=true" replace-with="log4j.additivity.SecurityLogger=false"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 68b4ce0..773753d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -293,6 +293,10 @@ </task> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hdfs NameNode"> + <task xsi:type="configure" id="hdfs_securitylogger_additivity"/> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index fb5e584..da5265d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -672,6 +672,7 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> + <task xsi:type="configure" id="hdfs_securitylogger_additivity" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 665e33d..3c309ce 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -261,6 +261,11 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + <definition xsi:type="configure" id="hdfs_securitylogger_additivity" summary="Set additivity of SecurityLogger to false"> + <type>hdfs-log4j</type> + <regex-replace key="content" find="hadoop.security.log.file=SecurityAuth.audit" replace-with="hadoop.security.log.file=SecurityAuth.audit log4j.additivity.SecurityLogger=false"/> + <regex-replace key="content" find="log4j.additivity.SecurityLogger=true" replace-with="log4j.additivity.SecurityLogger=false"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index a1bf456..eb2c18a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -265,6 +265,11 @@ <direction>UPGRADE</direction> <!-- prevent config changes on downgrade --> <skippable>true</skippable> <!-- May fix configuration problems manually --> + <!--HDFS--> + <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hdfs NameNode"> + <task xsi:type="configure" id="hdfs_securitylogger_additivity"/> + </execute-stage> + <!--YARN--> <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index cf97a72..10ae959 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -677,6 +677,7 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> + <task xsi:type="configure" id="hdfs_securitylogger_additivity" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 5d19652..c13724b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -230,6 +230,11 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + <definition xsi:type="configure" id="hdfs_securitylogger_additivity" summary="Set additivity of SecurityLogger to false"> + <type>hdfs-log4j</type> + <regex-replace key="content" find="hadoop.security.log.file=SecurityAuth.audit" replace-with="hadoop.security.log.file=SecurityAuth.audit log4j.additivity.SecurityLogger=false"/> + <regex-replace key="content" find="log4j.additivity.SecurityLogger=true" replace-with="log4j.additivity.SecurityLogger=false"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 9f7a01a..57a5203 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -276,6 +276,11 @@ <direction>UPGRADE</direction> <!-- prevent config changes on downgrade --> <skippable>true</skippable> <!-- May fix configuration problems manually --> + <!--HDFS--> + <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hdfs NameNode"> + <task xsi:type="configure" id="hdfs_securitylogger_additivity"/> + </execute-stage> + <!--YARN--> <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 97313b5..ab6b239 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -605,6 +605,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> + <task xsi:type="configure" id="hdfs_securitylogger_additivity" /> </pre-upgrade> <pre-downgrade /> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b7e900/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml index 777c846..4d0cd73 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HDFS/configuration/hdfs-log4j.xml @@ -115,6 +115,7 @@ hadoop.security.log.maxfilesize=256MB hadoop.security.log.maxbackupindex=20 log4j.category.SecurityLogger=${hadoop.security.logger} hadoop.security.log.file=SecurityAuth.audit +log4j.additivity.SecurityLogger=false log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
