Repository: ambari Updated Branches: refs/heads/branch-2.5 9b309e31a -> 5aa398c63
AMBARI-21610. EU: Modify configs to enable custom extensions functionality. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5aa398c6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5aa398c6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5aa398c6 Branch: refs/heads/branch-2.5 Commit: 5aa398c631808c08ab433bc34a2adda1e6cb6666 Parents: 9b309e3 Author: Siddharth Wagle <[email protected]> Authored: Fri Jul 28 17:04:58 2017 -0700 Committer: Siddharth Wagle <[email protected]> Committed: Fri Jul 28 17:47:45 2017 -0700 ---------------------------------------------------------------------- .../4.2.5/upgrades/config-upgrade.xml | 24 ++++++++++++++++++ .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 22 ++++++++++++++++- .../BigInsights/4.2/upgrades/config-upgrade.xml | 26 ++++++++++++++++++++ .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 22 ++++++++++++++++- 4 files changed, 92 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml index f71e91a..f4068a2 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml @@ -39,6 +39,7 @@ <type>hadoop-env</type> <replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" /> <replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} if [ -d "/usr/lib/hadoop-lzo" ]; then export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/* export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native fi"/> + <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append" newline-after="true" newline-before="true"/> </definition> <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments"> <type>hdfs-site</type> @@ -51,6 +52,10 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions"> + <type>core-site</type> + <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop" if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/> + </definition> </changes> </component> </service> @@ -99,6 +104,14 @@ <type>hbase-env</type> <set key="backup_wal_dir" value="true"/> </definition> + <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions" summary="Enable custom extension"> + <type>hbase-site</type> + <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase" if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/> + </definition> + <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions" summary="Enable custom extensions"> + <type>hbase-env</type> + <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional. export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/> + </definition> </changes> </component> </service> @@ -216,6 +229,17 @@ <type>hive-site</type> <set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode" if-key-state="absent"/> </definition> + + <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions" summary="Enable custom extensions in hive-site"> + <type>hive-site</type> + <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive" if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/> + </definition> + + <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions" summary="Enable custom extensions in hive-env"> + <type>hive-env</type> + <replace key="content" find="if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then"/> + <replace key="content" find="if [ -f "${HIVE_AUX_JARS_PATH}" ]; then" replace-with="if [ -f "${HIVE_AUX_JARS_PATH}" ] || [ -d "${HIVE_AUX_JARS_PATH}" ] ; then"/> + </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index a7fdeec..00e8b22 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -222,6 +222,12 @@ </task> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions in core-site"> + <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions"> + <summary>Enable custom extensions in core-site</summary> + </task> + </execute-stage> + <!-- core-site --> <execute-stage title="Fix auth_to_local mapping"> <condition xsi:type="security" type="kerberos"/> @@ -257,6 +263,14 @@ <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" /> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE"> + <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/> + </execute-stage> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE"> + <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/> + </execute-stage> + <!--RANGER--> <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger Admin"> <task xsi:type="configure" id="hdp_2_6_0_0_remove_bind_anonymous"/> @@ -304,7 +318,13 @@ </execute-stage> <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/> - </execute-stage> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server"> + <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server"> + <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/> + </execute-stage> <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server"> <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" /> </execute-stage> http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml index e8ccc22..17e80b5 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml @@ -131,6 +131,7 @@ <type>hadoop-env</type> <replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" /> <replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} if [ -d "/usr/lib/hadoop-lzo" ]; then export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/* export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native fi"/> + <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append" newline-after="true" newline-before="true"/> </definition> <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments"> <type>hdfs-site</type> @@ -143,6 +144,10 @@ <type>hadoop-env</type> <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions"> + <type>core-site</type> + <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop" if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/> + </definition> </changes> </component> </service> @@ -153,6 +158,7 @@ <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update Yarn configurations"> <type>yarn-site</type> <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/> + <insert key="yarn.application.classpath" insert-type="append" newline-after="false" newline-before="false" value=",/usr/hdp/current/ext/hadoop/*"/> </definition> <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs"> <type>yarn-env</type> @@ -168,6 +174,7 @@ <definition xsi:type="configure" id="biginsights_4_2_mapreduce_application_framework_patch" summary="Update MapReduce2 configurations"> <type>mapred-site</type> <set key="mapreduce.application.framework.path" value="/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework"/> + <insert key="mapreduce.application.classpath" insert-type="append" value=":/usr/hdp/current/ext/hadoop/*" newline-before="false" newline-after="false"/> </definition> </changes> </component> @@ -191,6 +198,14 @@ <type>hbase-env</type> <set key="backup_wal_dir" value="true"/> </definition> + <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions" summary="Enable custom extensions in hbase-site"> + <type>hbase-site</type> + <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase" if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/> + </definition> + <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions" summary="Enable custom extensions in hbase-env"> + <type>hbase-env</type> + <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional. export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/> + </definition> </changes> </component> </service> @@ -240,6 +255,17 @@ <type>hive-site</type> <set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode" if-key-state="absent"/> </definition> + + <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions" summary="Enable custom extensions in hive-site"> + <type>hive-site</type> + <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive" if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/> + </definition> + + <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions" summary="Enable custom extensions in hive-env"> + <type>hive-env</type> + <replace key="content" find="if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then"/> + <replace key="content" find="if [ -f "${HIVE_AUX_JARS_PATH}" ]; then" replace-with="if [ -f "${HIVE_AUX_JARS_PATH}" ] || [ -d "${HIVE_AUX_JARS_PATH}" ] ; then"/> + </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index ed6414d..55736cf 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -217,6 +217,12 @@ </task> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions in core-site"> + <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions"> + <summary>Enable custom extensions in core-site</summary> + </task> + </execute-stage> + <!-- core-site --> <execute-stage title="Fix auth_to_local mapping"> <condition xsi:type="security" type="kerberos"/> @@ -252,6 +258,14 @@ <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" /> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE"> + <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/> + </execute-stage> + + <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE"> + <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/> + </execute-stage> + <!-- HIVE --> <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="biginsights_4_2_hive_env_configure"/> @@ -261,7 +275,13 @@ </execute-stage> <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server"> <task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/> - </execute-stage> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server"> + <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/> + </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server"> + <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/> + </execute-stage> <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server"> <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" /> </execute-stage>
