This is an automated email from the ASF dual-hosted git repository.
swagle pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6f79c88 [AMBARI-23370] Download client configs fails due to
'clusterLevelParams' not found (#802)
6f79c88 is described below
commit 6f79c888aed062ef60d40bbd647d43f5629b0b2d
Author: majorendre <34535487+majoren...@users.noreply.github.com>
AuthorDate: Thu Mar 29 20:55:37 2018 +0200
[AMBARI-23370] Download client configs fails due to 'clusterLevelParams'
not found (#802)
---
.../server/controller/KerberosHelperImpl.java | 10 +---
.../internal/ClientConfigResourceProvider.java | 61 +++++++---------------
.../topology/ClusterConfigurationRequest.java | 5 +-
.../org/apache/ambari/server/utils/StageUtils.java | 61 +++++-----------------
.../controller/AmbariManagementControllerTest.java | 2 +-
.../apache/ambari/server/utils/StageUtilsTest.java | 24 ++++-----
6 files changed, 45 insertions(+), 118 deletions(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index d2323c4..3f0ea84 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -588,8 +588,6 @@ public class KerberosHelperImpl implements KerberosHelper {
configurations.put("clusterHostInfo", clusterHostInfoMap);
}
- Map<String, String> componentToClusterInfoMap =
StageUtils.getComponentToClusterInfoKeyMap();
-
// Iterate through the recommendations to find the recommended host
assignments
for (RecommendationResponse.HostGroup hostGroup : hostGroups) {
Set<Map<String, String>> components = hostGroup.getComponents();
@@ -607,13 +605,7 @@ public class KerberosHelperImpl implements KerberosHelper {
// If the component filter is null or the current component
is found in the filter,
// include it in the map
if ((componentFilter == null) ||
componentFilter.contains(componentName)) {
- String key = componentToClusterInfoMap.get(componentName);
-
- if (StringUtils.isEmpty(key)) {
- // If not found in the componentToClusterInfoMap, then
keys are assumed to be
- // in the form of <component_name>_hosts (lowercase)
- key = componentName.toLowerCase() + "_hosts";
- }
+ String key =
StageUtils.getClusterHostInfoKey(componentName);
Set<String> fqdns = new TreeSet<>();
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index fda3817..8992413 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -65,6 +65,7 @@ import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariManagementControllerImpl;
import org.apache.ambari.server.controller.MaintenanceStateHelper;
import org.apache.ambari.server.controller.ServiceComponentHostRequest;
import org.apache.ambari.server.controller.ServiceComponentHostResponse;
@@ -369,27 +370,13 @@ public class ClientConfigResourceProvider extends
AbstractControllerResourceProv
}
osFamily = clusters.getHost(hostName).getOsFamily();
- TreeMap<String, String> hostLevelParams = new TreeMap<>();
- StageUtils.useStackJdkIfExists(hostLevelParams, configs);
- hostLevelParams.put(JDK_LOCATION,
managementController.getJdkResourceUrl());
- hostLevelParams.put(STACK_NAME, stackId.getStackName());
- hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
- hostLevelParams.put(DB_NAME, managementController.getServerDB());
- hostLevelParams.put(MYSQL_JDBC_URL,
managementController.getMysqljdbcUrl());
- hostLevelParams.put(ORACLE_JDBC_URL,
managementController.getOjdbcUrl());
- hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped());
- hostLevelParams.putAll(managementController.getRcaParameters());
- hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY,
configs.isAgentStackRetryOnInstallEnabled());
- hostLevelParams.put(AGENT_STACK_RETRY_COUNT,
configs.getAgentStackRetryOnInstallCount());
- hostLevelParams.put(GPL_LICENSE_ACCEPTED,
configs.getGplLicenseAccepted().toString());
-
// Write down os specific info for the service
ServiceOsSpecific anyOs = null;
if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) {
anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS);
}
- ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo,
hostLevelParams, osFamily);
+ ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo,
osFamily);
// Build package list that is relevant for host
List<ServiceOsSpecific.Package> packages =
@@ -403,25 +390,6 @@ public class ClientConfigResourceProvider extends
AbstractControllerResourceProv
}
String packageList = gson.toJson(packages);
- Set<String> userSet =
configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER,
cluster, desiredClusterConfigs);
- String userList = gson.toJson(userSet);
- hostLevelParams.put(USER_LIST, userList);
-
- //Create a user_group mapping and send it as part of the
hostLevelParams
- Map<String, Set<String>> userGroupsMap =
configHelper.createUserGroupsMap(
- stackId, cluster, desiredClusterConfigs);
- String userGroups = gson.toJson(userGroupsMap);
- hostLevelParams.put(USER_GROUPS,userGroups);
-
- Set<String> groupSet =
configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP,
cluster, desiredClusterConfigs);
- String groupList = gson.toJson(groupSet);
- hostLevelParams.put(GROUP_LIST, groupList);
-
- Map<org.apache.ambari.server.state.PropertyInfo, String>
notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs);
- Set<String> notManagedHdfsPathSet =
configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap,
NOT_MANAGED_HDFS_PATH_LIST);
- String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
- hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST,
notManagedHdfsPathList);
-
String jsonConfigurations = null;
Map<String, Object> commandParams = new HashMap<>();
List<Map<String, String>> xmlConfigs = new LinkedList<>();
@@ -441,6 +409,18 @@ public class ClientConfigResourceProvider extends
AbstractControllerResourceProv
}
}
+ TreeMap<String, String> clusterLevelParams = null;
+ TreeMap<String, String> ambariLevelParams = null;
+ if (getManagementController() instanceof
AmbariManagementControllerImpl){
+ AmbariManagementControllerImpl controller =
((AmbariManagementControllerImpl)getManagementController());
+ clusterLevelParams =
controller.getMetadataClusterLevelParams(cluster, stackId);
+ ambariLevelParams = controller.getMetadataAmbariLevelParams();
+ }
+ TreeMap<String, String> agentLevelParams = new TreeMap<>();
+ agentLevelParams.put("hostname", hostName);
+ agentLevelParams.put("public_hostname", publicHostName);
+ agentLevelParams.put("agentCacheDir", "/var/lib/ambari-agent/cache");
+
commandParams.put(PACKAGE_LIST, packageList);
commandParams.put("xml_configs_list", xmlConfigs);
commandParams.put("env_configs_list", envConfigs);
@@ -452,7 +432,9 @@ public class ClientConfigResourceProvider extends
AbstractControllerResourceProv
jsonContent.put("configuration_attributes", configurationAttributes);
jsonContent.put("commandParams", commandParams);
jsonContent.put("clusterHostInfo", clusterHostInfo);
- jsonContent.put("hostLevelParams", hostLevelParams);
+ jsonContent.put("ambariLevelParams", ambariLevelParams);
+ jsonContent.put("clusterLevelParams", clusterLevelParams);
+ jsonContent.put("agentLevelParams", agentLevelParams);
jsonContent.put("hostname", hostName);
jsonContent.put("public_hostname", publicHostName);
jsonContent.put("clusterName", cluster.getClusterName());
@@ -926,20 +908,13 @@ public class ClientConfigResourceProvider extends
AbstractControllerResourceProv
}
- protected ServiceOsSpecific populateServicePackagesInfo(ServiceInfo
serviceInfo, Map<String, String> hostParams,
- String osFamily) {
+ protected ServiceOsSpecific populateServicePackagesInfo(ServiceInfo
serviceInfo, String osFamily) {
ServiceOsSpecific hostOs = new ServiceOsSpecific(osFamily);
List<ServiceOsSpecific> foundedOSSpecifics =
getOSSpecificsByFamily(serviceInfo.getOsSpecifics(), osFamily);
if (!foundedOSSpecifics.isEmpty()) {
for (ServiceOsSpecific osSpecific : foundedOSSpecifics) {
hostOs.addPackages(osSpecific.getPackages());
}
- // Choose repo that is relevant for host
- ServiceOsSpecific.Repo serviceRepo = hostOs.getRepo();
- if (serviceRepo != null) {
- String serviceRepoInfo = gson.toJson(serviceRepo);
- hostParams.put(SERVICE_REPO_INFO, serviceRepoInfo);
- }
}
return hostOs;
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 740dd91..92aecb3 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -285,10 +285,7 @@ public class ClusterConfigurationRequest {
for (String component : components) {
Collection<String> componentHost =
clusterTopology.getHostAssignmentsForComponent(component);
// retrieve corresponding clusterInfoKey for component using StageUtils
- String clusterInfoKey =
StageUtils.getComponentToClusterInfoKeyMap().get(component);
- if (clusterInfoKey == null) {
- clusterInfoKey = component.toLowerCase() + "_hosts";
- }
+ String clusterInfoKey = StageUtils.getClusterHostInfoKey(component);
componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost,
","));
}
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index b6287e6..1130e3d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -92,8 +92,6 @@ public class StageUtils {
protected static final String RACKS = "all_racks";
protected static final String IPV4_ADDRESSES = "all_ipv4_ips";
- private static Map<String, String> componentToClusterInfoKeyMap =
- new HashMap<>();
private static Map<String, String> decommissionedToClusterInfoKeyMap =
new HashMap<>();
private volatile static Gson gson;
@@ -152,36 +150,6 @@ public class StageUtils {
}
static {
- componentToClusterInfoKeyMap.put("NAMENODE", "namenode_host");
- componentToClusterInfoKeyMap.put("JOBTRACKER", "jtnode_host");
- componentToClusterInfoKeyMap.put("SECONDARY_NAMENODE", "snamenode_host");
- componentToClusterInfoKeyMap.put("RESOURCEMANAGER", "rm_host");
- componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts");
- componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host");
- componentToClusterInfoKeyMap.put("JOURNALNODE", "journalnode_hosts");
- componentToClusterInfoKeyMap.put("ZKFC", "zkfc_hosts");
- componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts");
- componentToClusterInfoKeyMap.put("FLUME_HANDLER", "flume_hosts");
- componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts");
- componentToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "hbase_rs_hosts");
- componentToClusterInfoKeyMap.put("HIVE_SERVER", "hive_server_host");
- componentToClusterInfoKeyMap.put("HIVE_METASTORE", "hive_metastore_host");
- componentToClusterInfoKeyMap.put("OOZIE_SERVER", "oozie_server");
- componentToClusterInfoKeyMap.put("WEBHCAT_SERVER", "webhcat_server_host");
- componentToClusterInfoKeyMap.put("MYSQL_SERVER", "hive_mysql_host");
- componentToClusterInfoKeyMap.put("DASHBOARD", "dashboard_host");
- componentToClusterInfoKeyMap.put("GANGLIA_SERVER", "ganglia_server_host");
- componentToClusterInfoKeyMap.put("DATANODE", "slave_hosts");
- componentToClusterInfoKeyMap.put("TASKTRACKER", "mapred_tt_hosts");
- componentToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "hbase_rs_hosts");
- componentToClusterInfoKeyMap.put("ACCUMULO_MASTER",
"accumulo_master_hosts");
- componentToClusterInfoKeyMap.put("ACCUMULO_MONITOR",
"accumulo_monitor_hosts");
- componentToClusterInfoKeyMap.put("ACCUMULO_GC", "accumulo_gc_hosts");
- componentToClusterInfoKeyMap.put("ACCUMULO_TRACER",
"accumulo_tracer_hosts");
- componentToClusterInfoKeyMap.put("ACCUMULO_TSERVER",
"accumulo_tserver_hosts");
- }
-
- static {
decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts");
decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts");
decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts");
@@ -192,10 +160,6 @@ public class StageUtils {
return requestId + "-" + stageId;
}
- public static Map<String, String> getComponentToClusterInfoKeyMap() {
- return componentToClusterInfoKeyMap;
- }
-
public static long[] getRequestStage(String actionId) {
String[] fields = actionId.split("-");
long[] requestStageIds = new long[2];
@@ -278,6 +242,16 @@ public class StageUtils {
return commandParams;
}
+ /**
+ * A helper method for generating keys for the clusterHostInfo section.
+ */
+ public static String getClusterHostInfoKey(String componentName){
+ if (componentName == null){
+ throw new IllegalArgumentException("Component name cannot be null");
+ }
+ return componentName.toLowerCase()+"_hosts";
+ }
+
public static Map<String, Set<String>> getClusterHostInfo(Cluster cluster)
throws AmbariException {
//Fill hosts and ports lists
Set<String> hostsSet = new LinkedHashSet<>();
@@ -325,21 +299,10 @@ public class StageUtils {
ServiceComponent serviceComponent = serviceComponentEntry.getValue();
String componentName = serviceComponent.getName();
- String roleName = componentToClusterInfoKeyMap.get(componentName);
- if(null == roleName) {
- roleName = additionalComponentToClusterInfoKeyMap.get(componentName);
- }
- if (null == roleName && !serviceComponent.isClientComponent()) {
- roleName = componentName.toLowerCase() + "_hosts";
- additionalComponentToClusterInfoKeyMap.put(componentName, roleName);
- }
+ String roleName = getClusterHostInfoKey(componentName);
String decomRoleName =
decommissionedToClusterInfoKeyMap.get(componentName);
- if (roleName == null && decomRoleName == null) {
- continue;
- }
-
for (String hostName :
serviceComponent.getServiceComponentHosts().keySet()) {
if (roleName != null) {
@@ -380,7 +343,7 @@ public class StageUtils {
Collection<String> hostComponents = entry.getValue();
for (String hostComponent : hostComponents) {
- String roleName = componentToClusterInfoKeyMap.get(hostComponent);
+ String roleName = getClusterHostInfoKey(hostComponent);
if (null == roleName) {
roleName = additionalComponentToClusterInfoKeyMap.get(hostComponent);
}
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 084efdd..af78e29 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -1318,7 +1318,7 @@ public class AmbariManagementControllerTest {
assertTrue(ec.getCommandParams().containsKey("command_retry_enabled"));
assertEquals("false", ec.getCommandParams().get("command_retry_enabled"));
Map<String, Set<String>> chInfo = ec.getClusterHostInfo();
- assertTrue(chInfo.containsKey("namenode_host"));
+ assertTrue(chInfo.containsKey("namenode_hosts"));
assertFalse(ec.getCommandParams().containsKey("custom_folder"));
ec = controller.getExecutionCommand(cluster,
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
index 3e8d348..8f3fe8e 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
@@ -559,19 +559,19 @@ public class StageUtilsTest extends EasyMockSupport {
assertEquals(StageUtils.getHostName(), serverHost.iterator().next());
// check host role replacing by the projected topology
- assertTrue(getDecompressedSet(info.get("hbase_rs_hosts")).contains(9));
+
assertTrue(getDecompressedSet(info.get("hbase_regionserver_hosts")).contains(9));
// Validate substitutions...
info = StageUtils.substituteHostIndexes(info);
- checkServiceHostNames(info, "DATANODE", "slave_hosts", projectedTopology);
- checkServiceHostNames(info, "NAMENODE", "namenode_host",
projectedTopology);
- checkServiceHostNames(info, "SECONDARY_NAMENODE", "snamenode_host",
projectedTopology);
- checkServiceHostNames(info, "HBASE_MASTER", "hbase_master_hosts",
projectedTopology);
- checkServiceHostNames(info, "HBASE_REGIONSERVER", "hbase_rs_hosts",
projectedTopology);
- checkServiceHostNames(info, "JOBTRACKER", "jtnode_host",
projectedTopology);
- checkServiceHostNames(info, "TASKTRACKER", "mapred_tt_hosts",
projectedTopology);
- checkServiceHostNames(info, "NONAME_SERVER", "noname_server_hosts",
projectedTopology);
+ checkServiceHostNames(info, "DATANODE", projectedTopology);
+ checkServiceHostNames(info, "NAMENODE", projectedTopology);
+ checkServiceHostNames(info, "SECONDARY_NAMENODE", projectedTopology);
+ checkServiceHostNames(info, "HBASE_MASTER", projectedTopology);
+ checkServiceHostNames(info, "HBASE_REGIONSERVER", projectedTopology);
+ checkServiceHostNames(info, "JOBTRACKER", projectedTopology);
+ checkServiceHostNames(info, "TASKTRACKER", projectedTopology);
+ checkServiceHostNames(info, "NONAME_SERVER", projectedTopology);
}
private void insertTopology(Map<String, Collection<String>>
projectedTopology, String componentName, Set<String> hostNames) {
@@ -700,7 +700,7 @@ public class StageUtilsTest extends EasyMockSupport {
}
// Determine the actual hosts for a given component...
- Set<String> hosts = info.get(mappedComponentName);
+ Set<String> hosts =
info.get(StageUtils.getClusterHostInfoKey(componentName));
if (hosts != null) {
actualHostsList.addAll(getDecompressedSet(hosts));
}
@@ -708,7 +708,7 @@ public class StageUtilsTest extends EasyMockSupport {
assertEquals(expectedHostsList, actualHostsList);
}
- private void checkServiceHostNames(Map<String, Set<String>> info, String
componentName, String mappedComponentName,
+ private void checkServiceHostNames(Map<String, Set<String>> info, String
componentName,
Map<String, Collection<String>>
serviceTopology) {
Set<String> expectedHostsList = new HashSet<>();
Set<String> actualHostsList = new HashSet<>();
@@ -721,7 +721,7 @@ public class StageUtilsTest extends EasyMockSupport {
}
// Determine the actual hosts for a given component...
- Set<String> hosts = info.get(mappedComponentName);
+ Set<String> hosts =
info.get(StageUtils.getClusterHostInfoKey(componentName));
if (hosts != null) {
actualHostsList.addAll(hosts);
}
--
To stop receiving notification emails like this one, please contact
swa...@apache.org.
