AMBARI-21627. Cross-stack upgrade from IOP to HDP, ranger audit properties need to be deleted (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/266deaf1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/266deaf1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/266deaf1 Branch: refs/heads/branch-feature-AMBARI-21450 Commit: 266deaf1dc0129ee447e9255ed253a16535e6bf5 Parents: f7a51bf Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Tue Aug 1 15:30:28 2017 -0700 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Wed Aug 2 18:14:15 2017 -0700 ---------------------------------------------------------------------- .../4.2.5/upgrades/config-upgrade.xml | 50 +++++++++++++++++ .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 18 +++++++ .../BigInsights/4.2/upgrades/config-upgrade.xml | 56 +++++++++++++++++++- .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 20 ++++++- 4 files changed, 141 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/266deaf1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml index d40d4d6..f6a21ff 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml @@ -71,6 +71,17 @@ <type>yarn-env</type> <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_yarn_audit_db"> + <type>ranger-yarn-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> </changes> </component> </service> @@ -112,6 +123,17 @@ <type>hbase-env</type> <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional. export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hbase_audit_db"> + <type>ranger-hbase-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> </changes> </component> </service> @@ -249,6 +271,17 @@ <replace key="content" find="if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then"/> <replace key="content" find="if [ -f "${HIVE_AUX_JARS_PATH}" ]; then" replace-with="if [ -f "${HIVE_AUX_JARS_PATH}" ] || [ -d "${HIVE_AUX_JARS_PATH}" ] ; then"/> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hive_audit_db"> + <type>ranger-hive-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> </changes> </component> @@ -301,5 +334,22 @@ </changes> </component> </service> + + <service name="KNOX"> + <component name="KNOX_GATEWAY"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_knox_audit_db"> + <type>ranger-knox-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> + </changes> + </component> + </service> </services> </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/266deaf1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index 609a4fe..361fb56 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -247,6 +247,10 @@ <task xsi:type="configure" id="biginsights_4_2_yarn_config_update" /> </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for YARN related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_yarn_audit_db"/> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs"> <task xsi:type="configure" id="yarn_env_security_opts"> @@ -277,6 +281,10 @@ <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBASE related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hbase_audit_db"/> + </execute-stage> + <!--RANGER--> <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger Admin"> <task xsi:type="configure" id="hdp_2_6_0_0_remove_bind_anonymous"/> @@ -334,6 +342,11 @@ <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server"> <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/> </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for HIVE related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hive_audit_db"/> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server"> <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" /> </execute-stage> @@ -356,6 +369,11 @@ <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes for Spark"> <task xsi:type="configure" id="hdp_2_6_0_0_spark2_defaults"/> </execute-stage> + + <!-- KNOX --> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for KNOX related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_knox_audit_db"/> + </execute-stage> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/266deaf1/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml index 9c9737f..f544e10 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml @@ -78,7 +78,7 @@ <service name="RANGER_KMS"> <component name="RANGER_KMS_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"> + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_audit_db"> <type>ranger-kms-audit</type> <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> @@ -88,6 +88,7 @@ <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> </definition> + <definition xsi:type="configure" id="kms_log4j_parameterize" summary="Parameterizing Ranger KMS Log4J Properties"> <type>kms-log4j</type> <set key="ranger_kms_log_maxfilesize" value="256"/> @@ -120,6 +121,7 @@ <set key="security.inter.broker.protocol" value="PLAINTEXTSASL"/> <replace key="listeners" find="SASL_PLAINTEXT" replace-with="PLAINTEXTSASL"/> </definition> + </changes> </component> </service> @@ -148,6 +150,7 @@ <type>core-site</type> <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop" if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/> </definition> + </changes> </component> </service> @@ -164,6 +167,17 @@ <type>yarn-env</type> <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_yarn_audit_db"> + <type>ranger-yarn-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> </changes> </component> </service> @@ -206,6 +220,17 @@ <type>hbase-env</type> <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional. export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hbase_audit_db"> + <type>ranger-hbase-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> </changes> </component> </service> @@ -275,6 +300,17 @@ <replace key="content" find="if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then"/> <replace key="content" find="if [ -f "${HIVE_AUX_JARS_PATH}" ]; then" replace-with="if [ -f "${HIVE_AUX_JARS_PATH}" ] || [ -d "${HIVE_AUX_JARS_PATH}" ] ; then"/> </definition> + + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hive_audit_db"> + <type>ranger-hive-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> </changes> </component> @@ -332,7 +368,23 @@ </changes> </component> </service> - </services> + <service name="KNOX"> + <component name="KNOX_GATEWAY"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_knox_audit_db"> + <type>ranger-knox-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> + </changes> + </component> + </service> + </services> </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/266deaf1/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index f409dd4..6e7c2e4 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -236,6 +236,10 @@ <task xsi:type="configure" id="biginsights_4_2_yarn_config_update" /> </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for YARN related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_yarn_audit_db"/> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs"> <task xsi:type="configure" id="yarn_env_security_opts"> @@ -266,6 +270,10 @@ <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/> </execute-stage> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBASE related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hbase_audit_db"/> + </execute-stage> + <!-- HIVE --> <execute-stage service="HIVE" component="HIVE_CLIENT" title="Update Hive Execution Engine"> <task xsi:type="configure" id="biginsights_4_2_hive_exectype_change"/> @@ -285,6 +293,11 @@ <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server"> <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/> </execute-stage> + + <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for HIVE related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_hive_audit_db"/> + </execute-stage> + <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server"> <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" /> </execute-stage> @@ -355,7 +368,7 @@ <!-- RANGER KMS --> <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS Server"> - <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"/> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_audit_db"/> </execute-stage> <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Parameterizing Ranger Kms Log4J Properties"> @@ -374,6 +387,11 @@ <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS"> <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl"/> </execute-stage> + + <!-- KNOX --> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for KNOX related to Ranger"> + <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_knox_audit_db"/> + </execute-stage> </group>
