Repository: ambari Updated Branches: refs/heads/branch-2.5 acb6e246f -> 1e72a0957
AMBARI-18699 - Upgrade Configuration Packs Should Have an XSD (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1e72a095 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1e72a095 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1e72a095 Branch: refs/heads/branch-2.5 Commit: 1e72a09574ced76488128172ef430fb0cbdd5214 Parents: acb6e24 Author: Jonathan Hurley <[email protected]> Authored: Wed Oct 26 08:18:10 2016 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Wed Oct 26 12:45:55 2016 -0400 ---------------------------------------------------------------------- .../upgrade/ConfigUpgradeChangeDefinition.java | 75 +-------- .../state/stack/upgrade/ConfigureTask.java | 49 +----- .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 110 +++++-------- .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 28 ++-- .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 29 ++-- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 4 +- .../src/main/resources/upgrade-config.xsd | 163 +++++++++++++++++++ .../ambari/server/state/UpgradeHelperTest.java | 50 +++--- .../state/stack/ConfigUpgradeValidityTest.java | 68 +++++++- .../HDP/2.1.1/upgrades/config-upgrade.xml | 52 ++---- .../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 2 +- .../HDP/2.2.0/upgrades/config-upgrade.xml | 19 +-- .../HDP/2.2.0/upgrades/config-upgrade.xml | 19 +-- 13 files changed, 353 insertions(+), 315 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java index 54431eb..5428ea7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java @@ -17,10 +17,9 @@ */ package org.apache.ambari.server.state.stack.upgrade; -import com.google.gson.Gson; -import org.apache.ambari.server.AmbariException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -28,9 +27,11 @@ import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlType; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; /** * The {@link ConfigUpgradeChangeDefinition} represents a configuration change. This change can be @@ -123,9 +124,6 @@ public class ConfigUpgradeChangeDefinition { @XmlElement(name = "set") private List<ConfigurationKeyValue> keyValuePairs; - @XmlElement(name = "condition") - private List<Condition> conditions; - @XmlElement(name = "transfer") private List<Transfer> transfers; @@ -147,13 +145,6 @@ public class ConfigUpgradeChangeDefinition { } /** - * @return the list of conditions - */ - public List<Condition> getConditions() { - return conditions; - } - - /** * @return the list of transfers, checking for appropriate null fields. */ public List<Transfer> getTransfers() { @@ -267,56 +258,6 @@ public class ConfigUpgradeChangeDefinition { } /** - * A conditional element that will only perform the configuration if the - * condition is met. - */ - @XmlAccessorType(XmlAccessType.FIELD) - @XmlType(name = "condition") - public static class Condition { - @XmlAttribute(name = "type") - private String conditionConfigType; - - @XmlAttribute(name = "key") - private String conditionKey; - - @XmlAttribute(name = "value") - private String conditionValue; - - @XmlElement(name = "type") - private String configType; - - @XmlElement(name = "key") - private String key; - - @XmlElement(name = "value") - private String value; - - public String getConditionConfigType() { - return conditionConfigType; - } - - public String getConditionKey() { - return conditionKey; - } - - public String getConditionValue() { - return conditionValue; - } - - public String getConfigType() { - return configType; - } - - public String getKey() { - return key; - } - - public String getValue() { - return value; - } - } - - /** * A {@code transfer} element will copy, move, or delete the value of one type/key to another type/key. */ @XmlAccessorType(XmlAccessType.FIELD) http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java index a9b355a..d7bb338 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java @@ -33,7 +33,6 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.stack.ConfigUpgradePack; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Condition; import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue; import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Replace; import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Transfer; @@ -192,40 +191,6 @@ public class ConfigureTask extends ServerSideActionTask { return configParameters; } - // the first matched condition will win; conditions make configuration tasks singular in - // the properties that can be set - when there is a condition the task will only contain - // conditions - List<Condition> conditions = definition.getConditions(); - if( null != conditions && !conditions.isEmpty() ){ - for (Condition condition : conditions) { - String conditionConfigType = condition.getConditionConfigType(); - String conditionKey = condition.getConditionKey(); - String conditionValue = condition.getConditionValue(); - - // always add the condition's target type just so that we have one to - // return even if none of the conditions match - configParameters.put(PARAMETER_CONFIG_TYPE, condition.getConfigType()); - - // check the condition; if it passes, set the configuration properties - // and break - String checkValue = getDesiredConfigurationValue(cluster, - conditionConfigType, conditionKey); - - if (conditionValue.equals(checkValue)) { - List<ConfigurationKeyValue> configurations = new ArrayList<>(1); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - keyValue.key = condition.getKey(); - keyValue.value = condition.getValue(); - configurations.add(keyValue); - - configParameters.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, - m_gson.toJson(configurations)); - - return configParameters; - } - } - } - // this task is not a condition task, so process the other elements normally if (null != definition.getConfigType()) { configParameters.put(PARAMETER_CONFIG_TYPE, definition.getConfigType()); @@ -260,8 +225,9 @@ public class ConfigureTask extends ServerSideActionTask { for(Replace replacement: replacements){ if(isValidConditionSettings(cluster, configType, replacement.key, - replacement.ifKey, replacement.ifType, replacement.ifValue, replacement.ifKeyState)) + replacement.ifKey, replacement.ifType, replacement.ifValue, replacement.ifKeyState)) { allowedReplacements.add(replacement); + } } return allowedReplacements; @@ -272,8 +238,9 @@ public class ConfigureTask extends ServerSideActionTask { for(ConfigurationKeyValue configurationKeyValue: sets){ if(isValidConditionSettings(cluster, configType, configurationKeyValue.key, - configurationKeyValue.ifKey, configurationKeyValue.ifType, configurationKeyValue.ifValue, configurationKeyValue.ifKeyState)) + configurationKeyValue.ifKey, configurationKeyValue.ifType, configurationKeyValue.ifValue, configurationKeyValue.ifKeyState)) { allowedSets.add(configurationKeyValue); + } } return allowedSets; @@ -283,14 +250,16 @@ public class ConfigureTask extends ServerSideActionTask { List<Transfer> allowedTransfers = new ArrayList<>(); for (Transfer transfer : transfers) { String key = ""; - if(transfer.operation == TransferOperation.DELETE) + if(transfer.operation == TransferOperation.DELETE) { key = transfer.deleteKey; - else + } else { key = transfer.fromKey; + } if(isValidConditionSettings(cluster, configType, key, - transfer.ifKey, transfer.ifType, transfer.ifValue, transfer.ifKeyState)) + transfer.ifKey, transfer.ifType, transfer.ifValue, transfer.ifKeyState)) { allowedTransfers.add(transfer); + } } return allowedTransfers; http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml index c225cca..6af8b43 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -16,8 +16,7 @@ limitations under the License. --> -<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-config.xsd"> <services> <service name="HDFS"> <component name="NAMENODE"> @@ -54,11 +53,12 @@ </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_hdfs_adjust_ranger_plugin"> - <condition type="ranger-hdfs-plugin-properties" key="ranger-hdfs-plugin-enabled" value="Yes"> - <type>hdfs-site</type> - <key>dfs.namenode.inode.attributes.provider.class</key> - <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value> - </condition> + <type>hdfs-site</type> + <set key="dfs.namenode.inode.attributes.provider.class" + value="org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer" + if-type="ranger-hdfs-plugin-properties" + if-key="ranger-hdfs-plugin-enabled" + if-value="Yes"/> </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_policy" @@ -446,16 +446,9 @@ </definition> <definition xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"> - <condition type="tez-site" key="tez.counters.max" value="2000"> - <type>tez-site</type> - <key>tez.counters.max</key> - <value>10000</value> - </condition> - <condition type="tez-site" key="tez.counters.max.groups" value="1000"> - <type>tez-site</type> - <key>tez.counters.max.groups</key> - <value>3000</value> - </condition> + <type>tez-site</type> + <set key="tez.counters.max" value="10000" if-type="tez-site" if-key="tez.counters.max" if-value="2000"/> + <set key="tez.counters.max.groups" value="3000" if-type="tez-site" if-key="tez.counters.max.groups" if-value="1000"/> </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"> @@ -493,11 +486,8 @@ </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_nimbus_monitor_freq_adjustment"> - <condition type="storm-site" key="nimbus.monitor.freq.secs" value="10"> - <type>storm-site</type> - <key>nimbus.monitor.freq.secs</key> - <value>120</value> - </condition> + <type>storm-site</type> + <set key="nimbus.monitor.freq.secs" value="120" if-type="storm-site" if-key="nimbus.monitor.freq.secs" if-value="10"/> </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds" @@ -699,19 +689,13 @@ </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_admin_hdfs_audit"> - <condition type="ranger-env" key="xasecure.audit.destination.hdfs" value="false"> - <type>ranger-env</type> - <key>xasecure.audit.destination.hdfs</key> - <value>false</value> - </condition> + <type>ranger-env</type> + <set key="xasecure.audit.destination.hdfs" value="false" if-type="ranger-env" if-key="xasecure.audit.destination.hdfs" if-value="false"/> </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_admin_db_audit"> - <condition type="ranger-env" key="xasecure.audit.destination.db" value="true"> - <type>ranger-env</type> - <key>xasecure.audit.destination.db</key> - <value>true</value> - </condition> + <type>ranger-env</type> + <set key="xasecure.audit.destination.db" value="true" if-type="ranger-env" if-key="xasecure.audit.destination.db" if-value="true"/> </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync" summary="Updating Ranger Usersync"> @@ -753,17 +737,14 @@ </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync_sync_source"> - <condition type="usersync-properties" key="SYNC_SOURCE" value="unix"> - <type>ranger-ugsync-site</type> - <key>ranger.usersync.source.impl.class</key> - <value>org.apache.ranger.unixusersync.process.UnixUserGroupBuilder</value> - </condition> + <type>ranger-ugsync-site</type> + <set key="ranger.usersync.source.impl.class" + value="org.apache.ranger.unixusersync.process.UnixUserGroupBuilder" + if-type="usersync-properties" if-key="SYNC_SOURCE" if-value="unix"/> - <condition type="usersync-properties" key="SYNC_SOURCE" value="ldap"> - <type>ranger-ugsync-site</type> - <key>ranger.usersync.source.impl.class</key> - <value>org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder</value> - </condition> + <set key="ranger.usersync.source.impl.class" + value="org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder" + if-type="usersync-properties" if-key="SYNC_SOURCE" if-value="ldap"/> </definition> <definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync_properties"> @@ -793,25 +774,18 @@ <service name="HBASE"> <component name="HBASE_MASTER"> <changes> - <definition xsi:type="configure" - id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory"> - <condition type="hbase-env" key="phoenix_sql_enabled" value="true"> - <type>hbase-site</type> - <key>hbase.region.server.rpc.scheduler.factory.class</key> - <value>org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory - </value> - </condition> + <definition xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory"> + <type>hbase-site</type> + <set key="hbase.region.server.rpc.scheduler.factory.class" + value="org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory" + if-type="hbase-env" if-key="phoenix_sql_enabled" if-value="true"/> </definition> - <definition xsi:type="configure" - id="hdp_2_3_0_0_hbase_master_adjust_phoenix_rpc_controller_factory"> - <condition type="hbase-env" key="phoenix_sql_enabled" value="true"> - <type>hbase-site</type> - <key>hbase.rpc.controllerfactory.class</key> - <value> - org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory - </value> - </condition> + <definition xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_rpc_controller_factory"> + <type>hbase-site</type> + <set key="hbase.rpc.controllerfactory.class" + value="org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory" + if-type="hbase-env" if-key="phoenix_sql_enabled" if-value="true"/> </definition> <definition xsi:type="configure" @@ -823,15 +797,11 @@ default-value="0.4"/> </definition> - <definition xsi:type="configure" - id="hdp_2_3_0_0_hbase_master_adjust_phoenix_indexed_wal_edit_codec"> - <condition type="hbase-env" key="phoenix_sql_enabled" value="true"> - <type>hbase-site</type> - <key>hbase.regionserver.wal.codec</key> - <value> - org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec - </value> - </condition> + <definition xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_indexed_wal_edit_codec"> + <type>hbase-site</type> + <set key="hbase.regionserver.wal.codec" + value="org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec" + if-type="hbase-env" if-key="phoenix_sql_enabled" if-value="true"/> </definition> <definition xsi:type="configure" @@ -1007,8 +977,7 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"> - <summary>Updating oozie-site to remove redundant configurations</summary> + <definition xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations" summary="Updating oozie-site to remove redundant configurations"> <type>oozie-site</type> <transfer operation="delete" delete-key="*" preserve-edits="true"> <keep-key>oozie.base.url</keep-key> @@ -1127,5 +1096,4 @@ </component> </service> </services> - </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index 916259b..fe1f494 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -16,8 +16,7 @@ limitations under the License. --> -<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-config.xsd"> <services> <service name="HBASE"> <component name="HBASE_MASTER"> @@ -100,16 +99,12 @@ <!-- Add these configs if the cluster is Kerberized. Will only be written to the local file system if Atlas is present. --> <definition xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs"> - <condition type="cluster-env" key="security_enabled" value="true"> - <type>sqoop-atlas-application.properties</type> - <key>atlas.jaas.KafkaClient.option.useTicketCache</key> - <value>true</value> - </condition> - <condition type="cluster-env" key="security_enabled" value="true"> - <type>sqoop-atlas-application.properties</type> - <key>atlas.jaas.KafkaClient.option.renewTicket</key> - <value>true</value> - </condition> + <type>sqoop-atlas-application.properties</type> + <set key="atlas.jaas.KafkaClient.option.useTicketCache" value="true" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + + <set key="atlas.jaas.KafkaClient.option.renewTicket" value="true" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> </definition> </changes> </component> @@ -228,11 +223,9 @@ </definition> <definition xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag"> - <condition type="ranger-env" key="is_solrCloud_enabled" value="true"> - <type>ranger-env</type> - <key>is_external_solrCloud_enabled</key> - <value>true</value> - </condition> + <type>ranger-env</type> + <set key="is_external_solrCloud_enabled" value="true" + if-type="ranger-env" if-key="is_solrCloud_enabled" if-value="true"/> </definition> </changes> @@ -490,5 +483,4 @@ </component> </service> </services> - </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 5a0c3ef..44c2a6e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -16,10 +16,8 @@ limitations under the License. --> -<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-config.xsd"> <services> - <service name="TEZ"> <component name="TEZ_CLIENT"> <changes> @@ -48,16 +46,12 @@ <!-- Add these configs if the cluster is Kerberized. Will only be written to the local file system if Atlas is present. --> <definition xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs"> - <condition type="cluster-env" key="security_enabled" value="true"> - <type>sqoop-atlas-application.properties</type> - <key>atlas.jaas.KafkaClient.option.useTicketCache</key> - <value>true</value> - </condition> - <condition type="cluster-env" key="security_enabled" value="true"> - <type>sqoop-atlas-application.properties</type> - <key>atlas.jaas.KafkaClient.option.renewTicket</key> - <value>true</value> - </condition> + <type>sqoop-atlas-application.properties</type> + <set key="atlas.jaas.KafkaClient.option.useTicketCache" value="true" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + + <set key="atlas.jaas.KafkaClient.option.renewTicket" value="true" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> </definition> </changes> </component> @@ -135,11 +129,9 @@ </definition> <definition xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag"> - <condition type="ranger-env" key="is_solrCloud_enabled" value="true"> - <type>ranger-env</type> - <key>is_external_solrCloud_enabled</key> - <value>true</value> - </condition> + <type>ranger-env</type> + <set key="is_external_solrCloud_enabled" value="true" + if-type="ranger-env" if-key="is_solrCloud_enabled" if-value="true"/> </definition> </changes> @@ -380,5 +372,4 @@ </service> </services> - </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 59e4ec5..87ede63 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -16,8 +16,7 @@ limitations under the License. --> -<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-config.xsd"> <services> <service name="STORM"> <component name="NIMBUS"> @@ -46,5 +45,4 @@ </component> </service> </services> - </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/main/resources/upgrade-config.xsd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade-config.xsd b/ambari-server/src/main/resources/upgrade-config.xsd new file mode 100644 index 0000000..e274451 --- /dev/null +++ b/ambari-server/src/main/resources/upgrade-config.xsd @@ -0,0 +1,163 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; version="1.1"> + <xs:annotation> + <xs:documentation> + This document describes the schema for Upgrade Pack configuration changes. + </xs:documentation> + </xs:annotation> + + <xs:simpleType name="transfer-operation-type"> + <xs:restriction base="xs:string"> + <xs:enumeration value="delete"/> + <xs:enumeration value="move"/> + <xs:enumeration value="copy"/> + </xs:restriction> + </xs:simpleType> + + <xs:simpleType name="set-if-key-state-type"> + <xs:restriction base="xs:string"> + <xs:enumeration value="present"/> + <xs:enumeration value="absent"/> + </xs:restriction> + </xs:simpleType> + + <xs:simpleType name="transfer-operation-coerce-type"> + <xs:restriction base="xs:string"> + <xs:enumeration value="yaml-array"/> + </xs:restriction> + </xs:simpleType> + + <xs:complexType name="configure"> + <xs:sequence> + <xs:element name="type" type="xs:string" minOccurs="1" maxOccurs="1"/> + <xs:choice minOccurs="0" maxOccurs="unbounded"> + <xs:element name="transfer" minOccurs="0" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="keep-key" type="xs:string" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + <xs:attribute name="operation" use="required" type="transfer-operation-type"/> + <xs:attribute name="from-type" use="optional" type="xs:string"/> + <xs:attribute name="from-key" use="optional" type="xs:string"/> + <xs:attribute name="to-key" use="optional" type="xs:string"/> + <xs:attribute name="delete-key" use="optional" type="xs:string"/> + <xs:attribute name="preserve-edits" use="optional" type="xs:boolean"/> + <xs:attribute name="default-value" use="optional" type="xs:string"/> + <xs:attribute name="coerce-to" use="optional" type="transfer-operation-coerce-type"/> + <xs:attribute name="if-key" use="optional" type="xs:string"/> + <xs:attribute name="if-type" use="optional" type="xs:string"/> + <xs:attribute name="if-value" use="optional" type="xs:string"/> + <xs:attribute name="if-key-state" use="optional" type="set-if-key-state-type"/> + <xs:attribute name="mask" use="optional" type="xs:boolean"/> + </xs:complexType> + </xs:element> + <xs:element name="set" minOccurs="0" maxOccurs="unbounded"> + <xs:complexType> + <xs:attribute name="key" use="required" type="xs:string"/> + <xs:attribute name="value" use="required" type="xs:string"/> + <xs:attribute name="if-key" use="optional" type="xs:string"/> + <xs:attribute name="if-type" use="optional" type="xs:string"/> + <xs:attribute name="if-value" use="optional" type="xs:string"/> + <xs:attribute name="if-key-state" use="optional" type="set-if-key-state-type"/> + <xs:attribute name="mask" use="optional" type="xs:boolean"/> + </xs:complexType> + </xs:element> + <xs:element name="replace" minOccurs="0" maxOccurs="unbounded"> + <xs:complexType> + <xs:attribute name="key" use="required" type="xs:string"/> + <xs:attribute name="find" use="required" type="xs:string"/> + <xs:attribute name="replace-with" use="required" type="xs:string"/> + <xs:attribute name="if-key" use="optional" type="xs:string"/> + <xs:attribute name="if-type" use="optional" type="xs:string"/> + <xs:attribute name="if-value" use="optional" type="xs:string"/> + <xs:attribute name="if-key-state" use="optional" type="set-if-key-state-type"/> + <xs:attribute name="mask" use="optional" type="xs:boolean"/> + </xs:complexType> + </xs:element> + </xs:choice> + </xs:sequence> + <xs:attribute name="id" use="required" type="xs:string"/> + <xs:attribute name="summary" use="optional" type="xs:string"/> + </xs:complexType> + + <xs:complexType name="changes-type"> + <xs:sequence> + <xs:element name="definition" type="configure" minOccurs="1" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + + <xs:complexType name="component-type"> + <xs:sequence> + <xs:element name="changes" type="changes-type" minOccurs="1" maxOccurs="1"> + <xs:unique name="unique-by-definition-id"> + <xs:annotation> + <xs:documentation>Ensures that the element "changes" does not have duplicate definitions</xs:documentation> + </xs:annotation> + <xs:selector xpath="definition"/> + <xs:field xpath="@id"/> + </xs:unique> + </xs:element> + </xs:sequence> + <xs:attribute name="name" use="required" type="xs:string"/> + </xs:complexType> + + <xs:complexType name="service-type"> + <xs:sequence> + <xs:element name="component" type="component-type" minOccurs="1" maxOccurs="unbounded"/> + </xs:sequence> + <xs:attribute name="name" use="required" type="xs:string"/> + </xs:complexType> + + <xs:complexType name="services-type"> + <xs:sequence> + <xs:element name="service" type="service-type" minOccurs="1" maxOccurs="unbounded"> + <xs:unique name="unique-by-component"> + <xs:annotation> + <xs:documentation>Ensures that the element "service" does not have duplicate components</xs:documentation> + </xs:annotation> + <xs:selector xpath="component" /> + <xs:field xpath="@name" /> + </xs:unique> + </xs:element> + </xs:sequence> + </xs:complexType> + + <xs:element name="upgrade-config-changes"> + <xs:annotation> + <xs:documentation> + This is the root element of an the configuration changes for an Upgrade Pack. + </xs:documentation> + </xs:annotation> + + <xs:complexType> + <xs:sequence> + <xs:element name="services" type="services-type" minOccurs="1"> + <xs:unique name="unique-by-service"> + <xs:annotation> + <xs:documentation>Ensures that the element "services" does not have duplicate services</xs:documentation> + </xs:annotation> + <xs:selector xpath="service" /> + <xs:field xpath="@name" /> + </xs:unique> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> +</xs:schema> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index a1d4c4b..d644a09 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -53,6 +53,7 @@ import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder; import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition; +import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue; import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.ExecuteTask; @@ -658,7 +659,7 @@ public class UpgradeHelperTest { // grab the configure task out of Hive UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0); + ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); // now change the thrift port to http to have the 2nd condition invoked Map<String, String> hiveConfigs = new HashMap<String, String>(); @@ -732,7 +733,8 @@ public class UpgradeHelperTest { assertEquals("HIVE", hiveGroup.name); //Condition is met - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(3).getTasks().get(0).getTasks().get(0); + ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get( + 0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); @@ -841,7 +843,7 @@ public class UpgradeHelperTest { assertEquals("HIVE", hiveGroup.name); //Condition is not met, so no config operations should be present in the configureTask... - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(4).getTasks().get(0).getTasks().get(0); + ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(3).getTasks().get(0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); @@ -853,7 +855,6 @@ public class UpgradeHelperTest { String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); - System.out.println(" testConfigTaskConditionSkip >> transferJson"+transferJson); String replacementJson = configProperties.get(ConfigureTask.PARAMETER_REPLACEMENTS); assertNotNull(configurationJson); @@ -876,6 +877,12 @@ public class UpgradeHelperTest { assertTrue(transfers.isEmpty()); } + /** + * Tests that {@link ConfigurationKeyValue} pairs on a {@link ConfigureTask} + * are correctly returned based on the if-conditions. + * + * @throws Exception + */ @Test public void testConfigureTask() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); @@ -894,30 +901,19 @@ public class UpgradeHelperTest { assertEquals(7, groups.size()); - // grab the configure task out of Hive + // grab the first configure task out of Hive UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get( - 0).getTasks().get(0); + ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); - assertNotNull(configurationJson); - - List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson(configurationJson, - new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() { - }.getType()); - - assertEquals("hive.server2.thrift.port", keyValuePairs.get(0).key); - assertEquals("10010", keyValuePairs.get(0).value); - - // now change the thrift port to http to have the 2nd condition invoked + // now set the property in the if-check in the set element so that we have a match Map<String, String> hiveConfigs = new HashMap<String, String>(); - hiveConfigs.put("hive.server2.transport.mode", "http"); - hiveConfigs.put("hive.server2.thrift.port", "10001"); + hiveConfigs.put("fooKey", "THIS-BETTER-CHANGE"); + hiveConfigs.put("ifFooKey", "ifFooValue"); ConfigurationRequest configurationRequest = new ConfigurationRequest(); configurationRequest.setClusterName(cluster.getClusterName()); configurationRequest.setType("hive-site"); @@ -935,20 +931,22 @@ public class UpgradeHelperTest { } }, null); - // the configure task should now return different properties + // the configure task should now return different properties to set based on + // the if-condition checks configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); assertEquals( configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); + String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); assertNotNull(configurationJson); - keyValuePairs = m_gson.fromJson(configurationJson, + List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson( + configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() { }.getType()); - assertEquals("hive.server2.http.port", keyValuePairs.get(0).key); - assertEquals("10011", keyValuePairs.get(0).value); + assertEquals("fooKey", keyValuePairs.get(0).key); + assertEquals("fooValue", keyValuePairs.get(0).value); } @Test @@ -970,7 +968,7 @@ public class UpgradeHelperTest { // grab the configure task out of Hive UpgradeGroupHolder hiveGroup = groups.get(4); assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0); + ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup); assertFalse(configProperties.isEmpty()); http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java index e764781..ee1b05e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java @@ -17,6 +17,8 @@ */ package org.apache.ambari.server.state.stack; +import java.io.File; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -26,6 +28,7 @@ import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.internal.UpgradeResourceProvider.ConfigurationPackBuilder; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; +import org.apache.ambari.server.stack.ModuleFileUnmarshaller; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent; @@ -36,6 +39,9 @@ import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.Grouping; import org.apache.ambari.server.state.stack.upgrade.Task; import org.apache.ambari.server.state.stack.upgrade.Task.Type; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.filefilter.IOFileFilter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -51,7 +57,8 @@ import junit.framework.Assert; /** * Tests that for every upgrade pack found, that all referenced configuration - * IDs exist in the {@code config-upgrade.xml} which will be used/created. + * IDs exist in the {@code config-upgrade.xml} which will be used/created. Also + * ensures that every XML file is valid against its XSD. */ @Category({ category.StackUpgradeTest.class}) public class ConfigUpgradeValidityTest { @@ -167,6 +174,65 @@ public class ConfigUpgradeValidityTest { Assert.assertTrue(validatedConfigCount > 100); } + @Test + @SuppressWarnings("unchecked") + public void testValidateConfigUpgradePacks() throws Exception { + IOFileFilter filter = new IOFileFilter() { + @Override + public boolean accept(File dir, String name) { + return false; + } + + @Override + public boolean accept(File file) { + // file has the folder named 'upgrades', ends with '.xml' and is NOT + // 'config-upgrade.xml' + if (file.getAbsolutePath().contains("upgrades") + && file.getAbsolutePath().endsWith("config-upgrade.xml")) { + return true; + } + + return false; + } + }; + + List<File> files = new ArrayList<>(); + + files.addAll(FileUtils.listFiles(new File("src/main/resources/stacks"), filter, + FileFilterUtils.directoryFileFilter())); + + files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks"), filter, + FileFilterUtils.directoryFileFilter())); + + files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks_with_upgrade_cycle"), + filter, FileFilterUtils.directoryFileFilter())); + + ModuleFileUnmarshaller unmarshaller = new ModuleFileUnmarshaller(); + + int filesTestedCount = 0; + for (File file : files) { + String fileContent = FileUtils.readFileToString(file, "UTF-8"); + + // these things must be in upgrade packs for them to work anyway + if (fileContent.contains("<upgrade-config-changes") + && fileContent.contains("xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"";)) { + if (!fileContent.contains("xsi:noNamespaceSchemaLocation=\"upgrade-config.xsd\"")) { + String msg = String.format( + "File %s appears to be a config upgrade pack, but does not define 'upgrade-config.xsd' as its schema", + file.getAbsolutePath()); + Assert.fail(msg); + } else { + filesTestedCount++; + unmarshaller.unmarshal(ConfigUpgradePack.class, file, true); + } + } + } + + Assert.assertTrue( + "This test didn't appear to do any work which could indicate that it failed to find files to validate", + filesTestedCount > 5); + } + /** * Asserts that an ID exists in a {@link ConfigUpgradePack}, throwing an * informative message if it does not. http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml index 44f9e02..307f4d4 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml @@ -16,13 +16,14 @@ limitations under the License. --> -<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-config.xsd"> <services> <service name="ZOOKEEPER"> <component name="ZOOKEEPER_SERVER"> <changes> <definition xsi:type="configure" id="hdp_2_1_1_zk_post_upgrade"> + <type>zookeeper-newconfig</type> + <set key="fooKey" value="fooValue"/> </definition> <definition xsi:type="configure" id="hdp_2_1_1_zookeeper_new_config_type"> @@ -48,8 +49,7 @@ <definition xsi:type="configure" id="hdp_2_1_1_nn_test"> <type>hdfs-site</type> - <key>myproperty</key> - <value>mynewvalue</value> + <set key="myproperty" value="mynewvalue"/> </definition> <definition xsi:type="configure" id="hdp_2_1_1_hdfs_new_config_type"> <type>hdfs-newconfig</type> @@ -83,35 +83,9 @@ <service name="HIVE"> <component name="HIVE_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_1_1_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - - <definition xsi:type="configure" id="hdp_2_1_1_test_properties"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_1_1_hive_server_foo"> <type>hive-site</type> - <set key="fooKey" value="fooValue"/> + <set key="fooKey" value="fooValue" if-type="hive-site" if-key="ifFooKey" if-value="ifFooValue"/> <set key="fooKey2" value="fooValue2"/> <set key="fooKey3" value="fooValue3"/> <transfer operation="copy" from-key="copy-key" to-key="copy-key-to" /> @@ -181,7 +155,7 @@ <!-- set --> <set key="setKeyOne" value="1" if-key="hive.server2.transport.mode" if-type="" if-value="skip"/> <set key="setKeyTwo" value="2" if-key="" if-type="hive-site" if-key-state="absent"/> - <set key="setKeyThree" value="3" if-key="foo.bar" if-type="hive-site" if-key-state="abcd"/> + <set key="setKeyThree" value="3" if-key="" if-type="hive-site" if-key-state="present"/> <set key="setKeyThree" value="3" if-key="foo.bar" if-type="hive-site" /> <!-- transfer operation Copy --> @@ -192,7 +166,7 @@ <transfer operation="copy" from-type="hive-site" from-key="copy-key-three" to-key="copy-to-key-four" default-value="1" if-key="hive.server2.transport.mode" if-type="hive-site" /> <transfer operation="copy" from-type="hive-site" from-key="copy-key-four" to-key="copy-to-key-four" default-value="1" - if-key="hive.server2.transport.mode" if-type="hive-site" if-key-state="abcd"/> + if-key="hive.server2.transport.mode" if-type="" if-key-state="present"/> <!-- transfer operation move --> <transfer operation="move" from-type="hive-site" from-key="move-key-one" to-key="move-to-key-four" default-value="1" @@ -200,14 +174,14 @@ <transfer operation="move" from-type="hive-site" from-key="move-key-two" to-key="move-to-key-two" default-value="1" if-key="" if-type="hive-site" if-key-state="absent"/> <transfer operation="move" from-type="hive-site" from-key="move-key-three" to-key="move-to-key-three" default-value="1" - if-key="hive.server2.transport.mode" if-type="hive-site" if-key-state="abcd"/> + if-key="hive.server2.transport.mode" if-type="" if-key-state="present"/> <transfer operation="move" from-type="hive-site" from-key="move-key-four" to-key="move-to-key-four" default-value="1" if-key="hive.server2.transport.mode" if-type="hive-site"/> <!-- transfer operation delete --> <transfer operation="delete" delete-key="delete-key-one" if-key="hive.server2.transport.mode" if-type="" if-key-state="absent"/> <transfer operation="delete" delete-key="delete-key-two" if-key="" if-type="hive-site" if-key-state="absent"/> - <transfer operation="delete" delete-key="delete-key-three" if-key="foo.bar" if-type="hive-site" if-key-state="abcd"/> + <transfer operation="delete" delete-key="delete-key-three" if-key="foo.bar" if-type="" if-key-state="present"/> <transfer operation="delete" delete-key="delete-key-four" if-key="hive.server2.transport.mode" if-type="hive-site" /> <!-- replacement --> @@ -216,13 +190,17 @@ <replace key="replace-key-two" find="efg" replace-with="efg-replaced" if-key="" if-type="hive-site" if-key-state="absent"/> <replace key="replace-key-three" find="ijk" replace-with="ijk-replaced" - if-key="foo.bar" if-type="hive-site" if-key-state="abcd"/> + if-key="foo.bar" if-type="" if-key-state="present"/> <replace key="replace-key-three" find="ijk" replace-with="ijk-replaced" if-key="foo.bar" if-type="hive-site"/> </definition> + + <definition xsi:type="configure" id="hdp_2_1_1_no_conditions_met"> + <type>hive-site</type> + <set key="fooKey" value="fooValue" if-type="hive-site" if-key="ifFooKey" if-value="ifFooValue"/> + </definition> </changes> </component> </service> </services> - </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml index 6dc143a..e15c24d 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml @@ -209,10 +209,10 @@ <summary>HiveServer Port Availability</summary> <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> </task> - <task xsi:type="configure" id="hdp_2_1_1_set_transport_mode"/> <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/> <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions"/> <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions_skip"/> + <task xsi:type="configure" id="hdp_2_1_1_no_conditions_met"/> </pre-upgrade> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml index 90d64b4..dc7465e 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml @@ -16,13 +16,14 @@ limitations under the License. --> -<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-config.xsd"> <services> <service name="ZOOKEEPER"> <component name="ZOOKEEPER_SERVER"> <changes> <definition xsi:type="configure" id="hdp_2_2_0_zk_post_upgrade"> + <type>zookeeper-newconfig</type> + <set key="fooKey" value="fooValue"/> </definition> </changes> </component> @@ -63,19 +64,6 @@ <service name="HIVE"> <component name="HIVE_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_2_0_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_2_0_hive_server_foo"> <type>hive-site</type> <set key="fooKey" value="fooValue"/> @@ -97,5 +85,4 @@ </component> </service> </services> - </upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/1e72a095/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml index 90d64b4..d40faba 100644 --- a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml +++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml @@ -16,13 +16,14 @@ limitations under the License. --> -<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> - +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-config.xsd"> <services> <service name="ZOOKEEPER"> <component name="ZOOKEEPER_SERVER"> <changes> <definition xsi:type="configure" id="hdp_2_2_0_zk_post_upgrade"> + <type>zookeeper-newconfig</type> + <set key="fooKey" value="fooValue"/> </definition> </changes> </component> @@ -63,19 +64,6 @@ <service name="HIVE"> <component name="HIVE_SERVER"> <changes> - <definition xsi:type="configure" id="hdp_2_2_0_set_transport_mode"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </definition> - <definition xsi:type="configure" id="hdp_2_2_0_hive_server_foo"> <type>hive-site</type> <set key="fooKey" value="fooValue"/> @@ -97,5 +85,4 @@ </component> </service> </services> - </upgrade-config-changes>
