AMBARI-21504. Restart of MR2 History Server failed due to null in immutable_paths.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/274a9951 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/274a9951 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/274a9951 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 274a9951c34721a867e8e541b4fa73bdf03aa5d0 Parents: f450eba Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Tue Jul 18 17:14:10 2017 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Tue Jul 18 17:14:10 2017 +0300 ---------------------------------------------------------------------- .../AmbariCustomCommandExecutionHelper.java | 7 +++++-- .../AmbariManagementControllerImpl.java | 3 ++- .../internal/ClientConfigResourceProvider.java | 3 ++- .../ambari/server/state/ConfigHelper.java | 17 +++++++++++++++ .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4 ++++ .../YARN/2.1.0.2.0/package/scripts/service.py | 4 ++++ .../AmbariManagementControllerImplTest.java | 12 ++++++++++- .../ambari/server/state/ConfigHelperTest.java | 22 ++++++++++++++++++++ 8 files changed, 67 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index aeb5a9c..5180870 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -88,6 +88,7 @@ import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.HostComponentAdminState; import org.apache.ambari.server.state.HostState; import org.apache.ambari.server.state.MaintenanceState; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.PropertyInfo.PropertyType; import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.Service; @@ -431,7 +432,8 @@ public class AmbariCustomCommandExecutionHelper { String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); @@ -1494,7 +1496,8 @@ public class AmbariCustomCommandExecutionHelper { hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 433ed56..38842fa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2522,7 +2522,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String groupList = gson.toJson(groupSet); hostParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties); + Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index 15c2d81..166fc5f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -407,7 +407,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); + Map<org.apache.ambari.server.state.PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index a3a676d..2a70ee1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -220,6 +220,23 @@ public class ConfigHelper { return resolved; } + + public Set<String> filterInvalidPropertyValues(Map<PropertyInfo, String> properties, String filteredListName) { + Set<String> resultSet = new HashSet<>(); + for (Iterator<Entry<PropertyInfo, String>> iterator = properties.entrySet().iterator(); iterator.hasNext();) { + Entry<PropertyInfo, String> property = iterator.next(); + PropertyInfo propertyInfo = property.getKey(); + String propertyValue = property.getValue(); + if (property == null || propertyValue == null || propertyValue.toLowerCase().equals("null") || propertyValue.isEmpty()) { + LOG.error(String.format("Excluding property %s from %s, because of invalid or empty value!", propertyInfo.getName(), filteredListName)); + iterator.remove(); + } else { + resultSet.add(propertyValue); + } + } + return resultSet; + } + /** * Get all config properties for a cluster given a set of configType to * versionTags map. This helper method merges all the override tags with a http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index c554349..897e6cb 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -96,6 +96,10 @@ class NameNode(Script): env.set_params(params) self.configure(env) hdfs_binary = self.get_hdfs_binary() + + if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null': + Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.") + namenode(action="start", hdfs_binary=hdfs_binary, upgrade_type=upgrade_type, upgrade_suspended=params.upgrade_suspended, env=env) http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py index 1c1b11b..7c59b60 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py @@ -26,6 +26,7 @@ from resource_management.libraries.functions.show_logs import show_logs from resource_management.libraries.functions.format import format from resource_management.core.resources.system import Execute, File from resource_management.core.signal_utils import TerminateStrategy +from resource_management.core.logger import Logger @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def service(componentName, action='start', serviceName='yarn'): @@ -43,6 +44,9 @@ def service(componentName, action='start', serviceName='yarn'): import params if serviceName == 'mapreduce' and componentName == 'historyserver': + if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null': + Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.") + delete_pid_file = True daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java index eadc678..1f2c332 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java @@ -2059,6 +2059,14 @@ public class AmbariManagementControllerImplTest { String JCE_NAME = "jceName"; String OJDBC_JAR_NAME = "OjdbcJarName"; String SERVER_DB_NAME = "ServerDBName"; + Map<PropertyInfo, String> notManagedHdfsPathMap = new HashMap<>(); + PropertyInfo propertyInfo1 = new PropertyInfo(); + propertyInfo1.setName("1"); + PropertyInfo propertyInfo2 = new PropertyInfo(); + propertyInfo2.setName("2"); + notManagedHdfsPathMap.put(propertyInfo1, "/tmp"); + notManagedHdfsPathMap.put(propertyInfo2, "/apps/falcon"); + Set<String> notManagedHdfsPathSet = new HashSet<>(Arrays.asList("/tmp", "/apps/falcon")); Gson gson = new Gson(); @@ -2089,8 +2097,10 @@ public class AmbariManagementControllerImplTest { expect(configuration.getPreviousDatabaseConnectorNames()).andReturn(new HashMap<String, String>()).anyTimes(); expect(repositoryVersionEntity.getVersion()).andReturn("1234").anyTimes(); expect(repositoryVersionEntity.getStackId()).andReturn(stackId).anyTimes(); - expect(configHelper.getPropertyValuesWithPropertyType(stackId, + expect(configHelper.getPropertiesWithPropertyType(stackId, PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs)).andReturn( + notManagedHdfsPathMap); + expect(configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST)).andReturn( notManagedHdfsPathSet); replay(manager, clusters, cluster, injector, stackId, configuration, repositoryVersionEntity, configHelper); http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java index 857da61..6e84b33 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java @@ -764,6 +764,28 @@ public class ConfigHelperTest { } @Test + public void testFilterInvalidPropertyValues() { + Map<PropertyInfo, String> properties = new HashMap<>(); + PropertyInfo prop1 = new PropertyInfo(); + prop1.setName("1"); + PropertyInfo prop2 = new PropertyInfo(); + prop1.setName("2"); + PropertyInfo prop3 = new PropertyInfo(); + prop1.setName("3"); + PropertyInfo prop4 = new PropertyInfo(); + prop1.setName("4"); + + properties.put(prop1, "/tmp"); + properties.put(prop2, "null"); + properties.put(prop3, ""); + properties.put(prop4, null); + + Set<String> resultSet = configHelper.filterInvalidPropertyValues(properties, "testlist"); + Assert.assertEquals(1, resultSet.size()); + Assert.assertEquals(resultSet.iterator().next(), "/tmp"); + } + + @Test public void testMergeAttributesWithNullProperties() throws Exception { Map<String, Map<String, String>> persistedAttributes = new HashMap<>(); Map<String, String> persistedFinalAttrs = new HashMap<>();
