AMBARI-22260. Update Spark2 log4j default settings to latest. (sai.sai.shao via sshridhar).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4042f9fd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4042f9fd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4042f9fd Branch: refs/heads/branch-feature-AMBARI-22008 Commit: 4042f9fde14548be77a5ed5daf45364df307c08a Parents: 0936f46 Author: Swapan Shridhar <[email protected]> Authored: Wed Nov 15 16:21:31 2017 -0800 Committer: Swapan Shridhar <[email protected]> Committed: Wed Nov 15 16:21:31 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 23 ++++++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 23 +++++--- .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 45 +++++++++------- .../configuration/spark2-log4j-properties.xml | 56 ++++++++++++++++++++ 4 files changed, 122 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index b1e5f91..8c2c860 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -104,14 +104,37 @@ </service> <service name="SPARK2"> + <component name="SPARK2_JOBHISTORYSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"> + <type>spark2-log4j-properties</type> + <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/> + <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/> + </definition> + </changes> + </component> <component name="SPARK2_CLIENT"> <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"> + <type>spark2-log4j-properties</type> + <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/> + <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/> + </definition> <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"> <type>spark2-defaults</type> <set key="spark.yarn.queue" value="default" if-type="spark-defaults" if-key="spark.yarn.queue" if-key-state="absent"/> </definition> </changes> </component> + <component name="SPARK2_THRIFTSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"> + <type>spark2-log4j-properties</type> + <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/> + <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/> + </definition> + </changes> + </component> </service> <service name="TEZ"> http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index ace9542..9caa850 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -77,7 +77,7 @@ <service name="FLUME"> <component>FLUME_HANDLER</component> </service> - + <service name="ACCUMULO"> <component>ACCUMULO_TRACER</component> <component>ACCUMULO_GC</component> @@ -355,7 +355,7 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixCapacitySchedulerOrderingPolicy"> <summary>Validate Root Queue Ordering Policy</summary> </task> - </execute-stage> + </execute-stage> <!--TEZ--> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez"> @@ -679,6 +679,15 @@ <summary>Add queue customization property</summary> </task> </execute-stage> + <execute-stage service="SPARK2" component="SPARK2_THRIFTSERVER" title="Apply config changes for Spark2 ThriftServer log4j"> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"/> + </execute-stage> + <execute-stage service="SPARK2" component="SPARK2_JOBHISTORYSERVER" title="Apply config changes for Spark2 Job HistoryServer log4j"> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"/> + </execute-stage> + <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes for Spark2 Client log4j"> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"/> + </execute-stage> </group> <!-- @@ -720,7 +729,7 @@ </task> </execute-stage> </group> - + <!-- Now, restart all of the services. --> <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper"> <service-check>false</service-check> @@ -1083,7 +1092,7 @@ <component>FLUME_HANDLER</component> </service> </group> - + <group xsi:type="restart" name="ACCUMULO" title="Accumulo"> <service-check>false</service-check> <skippable>true</skippable> @@ -1112,7 +1121,7 @@ <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check"> <direction>UPGRADE</direction> - + <execute-stage title="Check Component Versions"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" /> </execute-stage> @@ -1192,7 +1201,7 @@ <function>setup_ranger_java_patches</function> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> @@ -1215,7 +1224,7 @@ <function>configure_atlas_user_for_tagsync</function> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index df11ae1..4fc9b42 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -183,7 +183,7 @@ <component>HBASE_MASTER</component> </service> </group> - + <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check"> <direction>UPGRADE</direction> <skippable>true</skippable> @@ -199,7 +199,7 @@ <service>LOGSEARCH</service> </exclude> </group> - + <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated"> <service-check>false</service-check> <skippable>true</skippable> @@ -207,7 +207,7 @@ <component>DATANODE</component> <component>NFS_GATEWAY</component> </service> - + <service name="HBASE"> <component>HBASE_REGIONSERVER</component> <component>PHOENIX_QUERY_SERVER</component> @@ -223,7 +223,7 @@ <message>The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services.</message> </batch> </group> - + <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check"> <direction>UPGRADE</direction> <skippable>true</skippable> @@ -458,7 +458,7 @@ <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check"> <direction>UPGRADE</direction> - + <execute-stage title="Check Component Versions"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" /> </execute-stage> @@ -467,7 +467,7 @@ <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> <skippable>true</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> - + <execute-stage title="Confirm Finalize"> <direction>UPGRADE</direction> <task xsi:type="manual"> @@ -491,7 +491,7 @@ <function>finalize_rolling_upgrade</function> </task> </execute-stage> - + <execute-stage title="Save Cluster State"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction"> </task> @@ -577,7 +577,7 @@ <function>configure_atlas_user_for_tagsync</function> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> @@ -668,7 +668,7 @@ </upgrade> </component> </service> - + <service name="SLIDER"> <component name="SLIDER"> <upgrade> @@ -691,7 +691,7 @@ </task> <task xsi:type="configure" id="hdp_2_6_0_0_mapreduce_job_queuename"/> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> @@ -722,11 +722,11 @@ <task xsi:type="configure" id="yarn_site_retained_log_count" /> <task xsi:type="configure" id="hdp_2_6_0_0_service_check_queue_name"/> <task xsi:type="configure" id="hdp_2_6_0_0_ats_scan_interval_default"/> - + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixCapacitySchedulerOrderingPolicy"> <summary>Validate Root Queue Ordering Policy</summary> </task> - + </pre-upgrade> <pre-downgrade /> <upgrade> @@ -785,9 +785,9 @@ </task> <task xsi:type="configure" id="hdp_2_5_0_0_tez_queue_name"/> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> - + <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -865,9 +865,9 @@ <task xsi:type="configure" id="hdp_2_6_maint_llap_config"/> <task xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable"/> </pre-upgrade> - + <pre-downgrade /> - + <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -913,9 +913,9 @@ <task xsi:type="configure" id="hdp_2_5_0_0_rename_spark_livy_configs" /> <task xsi:type="configure" id="hdp_2_5_0_0_add_spark_conf_dir_livy_configs" /> </pre-upgrade> - + <pre-downgrade /> - + <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -933,11 +933,19 @@ <service name="SPARK2"> <component name="SPARK2_JOBHISTORYSERVER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"/> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> </component> <component name="SPARK2_THRIFTSERVER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"/> + </pre-upgrade> + <pre-downgrade /> <upgrade> <task xsi:type="restart-task" /> </upgrade> @@ -945,6 +953,7 @@ <component name="SPARK2_CLIENT"> <pre-upgrade> <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"/> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"/> </pre-upgrade> <pre-downgrade/> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml new file mode 100644 index 0000000..d7479f4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <description>Spark2-log4j-Properties</description> + <value> +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +# Set the default spark-shell log level to WARN. When running the spark-shell, the +# log level for this class is used to overwrite the root logger's log level, so that +# the user can have different defaults for the shell and regular Spark apps. +log4j.logger.org.apache.spark.repl.Main=WARN + +# Settings to quiet third party logs that are too verbose +log4j.logger.org.spark_project.jetty=WARN +log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR +log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO +log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO +log4j.logger.org.apache.parquet=ERROR +log4j.logger.parquet=ERROR + +# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support +log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL +log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration>
