Repository: ambari Updated Branches: refs/heads/branch-2.2 b73eab2ad -> 9b42aeba1 refs/heads/trunk 9e2a6030b -> 314bbc6b8
AMBARI-14390. EU from HDP 2.1 to 2.3 added NULL properties (afernandez via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9b42aeba Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9b42aeba Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9b42aeba Branch: refs/heads/branch-2.2 Commit: 9b42aeba14cc8bca06f38beaa24154d47b5386f7 Parents: b73eab2 Author: Lisnichenko Dmitro <[email protected]> Authored: Wed Dec 16 09:59:39 2015 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Wed Dec 16 09:59:39 2015 +0200 ---------------------------------------------------------------------- .../internal/UpgradeResourceProvider.java | 11 +++++++ .../ambari/server/state/ConfigMergeHelper.java | 4 +++ .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 14 ++++++++ .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 34 ++++++++++++++++++++ 4 files changed, 63 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9b42aeba/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 99a10f8..d2c5e1e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1025,6 +1025,17 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider if (null == newDefaultConfigurations) { newConfigurationsByType.put(configurationType, existingConfigurations); continue; + } else { + // TODO, should we remove existing configs whose value is NULL even though they don't have a value in the new stack? + + // Remove any configs in the new stack whose value is NULL, unless they currently exist and the value is not NULL. + Iterator<Map.Entry<String, String>> iter = newDefaultConfigurations.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry<String, String> entry = iter.next(); + if (entry.getValue() == null) { + iter.remove(); + } + } } // for every existing configuration, see if an entry exists; if it does http://git-wip-us.apache.org/repos/asf/ambari/blob/9b42aeba/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java index 0610137..5521a98 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java @@ -58,6 +58,7 @@ public class ConfigMergeHelper { Map<String, Map<String, String>> oldMap = new HashMap<String, Map<String, String>>(); Map<String, Map<String, String>> newMap = new HashMap<String, Map<String, String>>(); + // Add service properties for old and new stack for (String serviceName : cluster.getServices().keySet()) { Set<PropertyInfo> oldStackProperties = m_ambariMetaInfo.get().getServiceProperties( oldStack.getStackName(), oldStack.getStackVersion(), serviceName); @@ -68,6 +69,7 @@ public class ConfigMergeHelper { addToMap(newMap, newStackProperties); } + // Add stack properties for old and new stack Set<PropertyInfo> set = m_ambariMetaInfo.get().getStackProperties( oldStack.getStackName(), oldStack.getStackVersion()); addToMap(oldMap, set); @@ -76,6 +78,7 @@ public class ConfigMergeHelper { targetStack.getStackName(), targetStack.getStackVersion()); addToMap(newMap, set); + // Final result after merging. Map<String, Map<String, ThreeWayValue>> result = new HashMap<String, Map<String, ThreeWayValue>>(); @@ -97,6 +100,7 @@ public class ConfigMergeHelper { customValueKeys = CollectionUtils.subtract(valueKeys, oldPairs.keySet()); } + // Keep properties with custom values (i.e., changed from default value in old stack) if (null != customValueKeys) { for (String prop : customValueKeys) { String newVal = newPairs.get(prop); http://git-wip-us.apache.org/repos/asf/ambari/blob/9b42aeba/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml index fff9c97..2ff4662 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml @@ -221,11 +221,25 @@ <supports-auto-skip-failure>false</supports-auto-skip-failure> <!--Changes for stack 2.2--> + <!--HDFS--> + <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode without Ranger"> + <task xsi:type="configure" id="hdp_2_2_0_0_namenode_no_ranger"/> + </execute-stage> + <!--YARN--> <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer"> <task xsi:type="configure" id="hdp_2_2_0_0_historyserver_classpath"/> </execute-stage> + <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for YARN without Ranger"> + <task xsi:type="configure" id="hdp_2_2_0_0_historyserver_no_ranger"/> + </execute-stage> + + <!--STORM--> + <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus without Ranger"> + <task xsi:type="configure" id="hdp_2_2_0_0_nimbus_no_ranger"/> + </execute-stage> + <!--Changes for stack 2.3--> <!--HDFS--> <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode"> http://git-wip-us.apache.org/repos/asf/ambari/blob/9b42aeba/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml index ab384db..c2844fa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -19,6 +19,20 @@ <upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> <services> + + <service name="HDFS"> + <component name="NAMENODE"> + <changes> + <definition xsi:type="configure" id="hdp_2_2_0_0_namenode_no_ranger" summary="NameNode properties without Ranger"> + <!-- In HDP 2.1, RANGER was not present, so this property must not be added even though it is new in the + HDP 2.2 stack. The property is only valid in HDP 2.2 and higher if RANGER is present. --> + <type>hdfs-site</type> + <transfer operation="delete" delete-key="dfs.namenode.inode.attributes.provider.class"/> + </definition> + </changes> + </component> + </service> + <service name="YARN"> <component name="HISTORY_SERVER"> <changes> @@ -26,6 +40,13 @@ <type>yarn-site</type> <set key="yarn.application.classpath" value="$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*"/> </definition> + + <definition xsi:type="configure" id="hdp_2_2_0_0_historyserver_no_ranger" summary="YARN properties without Ranger"> + <!-- In HDP 2.1, RANGER was not present, so this property must not be added even though it is new in the + HDP 2.2 stack. The property is only valid in HDP 2.2 and higher if RANGER is present. --> + <type>yarn-site</type> + <transfer operation="delete" delete-key="yarn.authorization-provider"/> + </definition> </changes> </component> </service> @@ -91,4 +112,17 @@ </service> </services> + <service name="STORM"> + <component name="NIMBUS"> + <changes> + <definition xsi:type="configure" id="hdp_2_2_0_0_nimbus_no_ranger" summary="Nimbus properties without Ranger"> + <!-- In HDP 2.1, RANGER was not present, so this property must not be added even though it is new in the + HDP 2.2 stack. The property is only valid in HDP 2.2 and higher if RANGER is present. --> + <type>storm-site</type> + <transfer operation="delete" delete-key="nimbus.authorizer"/> + </definition> + </changes> + </component> + </service> + </upgrade-config-changes>
