Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-21450 81035d86c -> 23db89b4a
AMBARI-21504. Restart of MR2 History Server failed due to null in immutable_paths.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74cab506 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74cab506 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74cab506 Branch: refs/heads/branch-feature-AMBARI-21450 Commit: 74cab506783515f8b6ce62936b990277a6d26a95 Parents: d8a5bad Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Tue Jul 18 16:46:54 2017 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Tue Jul 18 16:46:54 2017 +0300 ---------------------------------------------------------------------- .../AmbariCustomCommandExecutionHelper.java | 7 +++++-- .../AmbariManagementControllerImpl.java | 3 ++- .../internal/ClientConfigResourceProvider.java | 3 ++- .../ambari/server/state/ConfigHelper.java | 17 +++++++++++++++ .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4 ++++ .../YARN/2.1.0.2.0/package/scripts/service.py | 4 ++++ .../AmbariManagementControllerImplTest.java | 12 ++++++++++- .../ambari/server/state/ConfigHelperTest.java | 22 ++++++++++++++++++++ 8 files changed, 67 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index e321559..2eca76c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -90,6 +90,7 @@ import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.HostComponentAdminState; import org.apache.ambari.server.state.HostState; import org.apache.ambari.server.state.MaintenanceState; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.PropertyInfo.PropertyType; import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.RepositoryVersionState; @@ -421,7 +422,8 @@ public class AmbariCustomCommandExecutionHelper { String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); @@ -1342,7 +1344,8 @@ public class AmbariCustomCommandExecutionHelper { hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index a34422d..b80ee98 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2541,7 +2541,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String groupList = gson.toJson(groupSet); hostParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties); + Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index daca2d0..fb90e15 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -409,7 +409,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv String groupList = gson.toJson(groupSet); hostLevelParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); + Map<org.apache.ambari.server.state.PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); + Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index ab8026c..a4c5511 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -220,6 +220,23 @@ public class ConfigHelper { return resolved; } + + public Set<String> filterInvalidPropertyValues(Map<PropertyInfo, String> properties, String filteredListName) { + Set<String> resultSet = new HashSet<>(); + for (Iterator<Entry<PropertyInfo, String>> iterator = properties.entrySet().iterator(); iterator.hasNext();) { + Entry<PropertyInfo, String> property = iterator.next(); + PropertyInfo propertyInfo = property.getKey(); + String propertyValue = property.getValue(); + if (property == null || propertyValue == null || propertyValue.toLowerCase().equals("null") || propertyValue.isEmpty()) { + LOG.error(String.format("Excluding property %s from %s, because of invalid or empty value!", propertyInfo.getName(), filteredListName)); + iterator.remove(); + } else { + resultSet.add(propertyValue); + } + } + return resultSet; + } + /** * Get all config properties for a cluster given a set of configType to * versionTags map. This helper method merges all the override tags with a http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 9a9f9ca..7f7e30c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -96,6 +96,10 @@ class NameNode(Script): env.set_params(params) self.configure(env) hdfs_binary = self.get_hdfs_binary() + + if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null': + Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.") + namenode(action="start", hdfs_binary=hdfs_binary, upgrade_type=upgrade_type, upgrade_suspended=params.upgrade_suspended, env=env) http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py index 6495209..deb9e64 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py @@ -25,6 +25,7 @@ from ambari_commons import OSConst from resource_management.core.shell import as_user, as_sudo from resource_management.libraries.functions.show_logs import show_logs from resource_management.core.signal_utils import TerminateStrategy +from resource_management.core.logger import Logger @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def service(componentName, action='start', serviceName='yarn'): @@ -42,6 +43,9 @@ def service(componentName, action='start', serviceName='yarn'): import params if serviceName == 'mapreduce' and componentName == 'historyserver': + if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null': + Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.") + delete_pid_file = True daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java index dbfee41..085b746 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java @@ -2055,6 +2055,14 @@ public class AmbariManagementControllerImplTest { String JCE_NAME = "jceName"; String OJDBC_JAR_NAME = "OjdbcJarName"; String SERVER_DB_NAME = "ServerDBName"; + Map<PropertyInfo, String> notManagedHdfsPathMap = new HashMap<>(); + PropertyInfo propertyInfo1 = new PropertyInfo(); + propertyInfo1.setName("1"); + PropertyInfo propertyInfo2 = new PropertyInfo(); + propertyInfo2.setName("2"); + notManagedHdfsPathMap.put(propertyInfo1, "/tmp"); + notManagedHdfsPathMap.put(propertyInfo2, "/apps/falcon"); + Set<String> notManagedHdfsPathSet = new HashSet<>(Arrays.asList("/tmp", "/apps/falcon")); Gson gson = new Gson(); @@ -2088,8 +2096,10 @@ public class AmbariManagementControllerImplTest { expect(clusterVersionDAO.findByClusterAndStateCurrent(clusterName)).andReturn(clusterVersionEntity).anyTimes(); expect(clusterVersionEntity.getRepositoryVersion()).andReturn(repositoryVersionEntity).anyTimes(); expect(repositoryVersionEntity.getVersion()).andReturn("1234").anyTimes(); - expect(configHelper.getPropertyValuesWithPropertyType(stackId, + expect(configHelper.getPropertiesWithPropertyType(stackId, PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs)).andReturn( + notManagedHdfsPathMap); + expect(configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST)).andReturn( notManagedHdfsPathSet); replay(manager, clusters, cluster, injector, stackId, configuration, clusterVersionDAO, clusterVersionEntity, http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java index b1c10f5..435b246 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java @@ -753,6 +753,28 @@ public class ConfigHelperTest { } @Test + public void testFilterInvalidPropertyValues() { + Map<PropertyInfo, String> properties = new HashMap<>(); + PropertyInfo prop1 = new PropertyInfo(); + prop1.setName("1"); + PropertyInfo prop2 = new PropertyInfo(); + prop1.setName("2"); + PropertyInfo prop3 = new PropertyInfo(); + prop1.setName("3"); + PropertyInfo prop4 = new PropertyInfo(); + prop1.setName("4"); + + properties.put(prop1, "/tmp"); + properties.put(prop2, "null"); + properties.put(prop3, ""); + properties.put(prop4, null); + + Set<String> resultSet = configHelper.filterInvalidPropertyValues(properties, "testlist"); + Assert.assertEquals(1, resultSet.size()); + Assert.assertEquals(resultSet.iterator().next(), "/tmp"); + } + + @Test public void testMergeAttributesWithNullProperties() throws Exception { Map<String, Map<String, String>> persistedAttributes = new HashMap<>(); Map<String, String> persistedFinalAttrs = new HashMap<>();