This is an automated email from the ASF dual-hosted git repository.
more pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new f1cec12 KNOX-2213 - Service Discovery Support for CM UI, API (#249)
f1cec12 is described below
commit f1cec12e720cdedb398ae050f60f98577dd01051
Author: Sandeep Moré <[email protected]>
AuthorDate: Mon Feb 3 12:16:35 2020 -0500
KNOX-2213 - Service Discovery Support for CM UI, API (#249)
---
.../cm/ClouderaManagerServiceDiscovery.java | 42 +++++++++-
.../ClouderaManagerAPIServiceModelGenerator.java | 94 ++++++++++++++++++++++
.../cm/ClouderaManagerUIServiceModelGenerator.java | 37 +++++++++
...way.topology.discovery.cm.ServiceModelGenerator | 2 +
.../cm/ClouderaManagerServiceDiscoveryTest.java | 32 +++++++-
5 files changed, 202 insertions(+), 5 deletions(-)
diff --git
a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
index aa66fae..a905677 100644
---
a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
+++
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
@@ -44,6 +44,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
@@ -68,6 +69,9 @@ public class ClouderaManagerServiceDiscovery implements
ServiceDiscovery {
static final String DEFAULT_USER_ALIAS = "cm.discovery.user";
static final String DEFAULT_PWD_ALIAS = "cm.discovery.password";
+ public static final String CM_SERVICE_TYPE = "CM";
+ public static final String CM_ROLE_TYPE = "CM_SERVER";
+
private static Map<String, List<ServiceModelGenerator>>
serviceModelGenerators = new HashMap<>();
static {
ServiceLoader<ServiceModelGenerator> loader =
ServiceLoader.load(ServiceModelGenerator.class);
@@ -216,18 +220,38 @@ public class ClouderaManagerServiceDiscovery implements
ServiceDiscovery {
Set<ServiceModel> serviceModels = new HashSet<>();
ApiServiceList serviceList = getClusterServices(servicesResourceApi,
clusterName);
+
if (serviceList != null) {
+ /*
+ Since Cloudera Manager does not have a service for itself, we will add a
skeleton CM
+ service so that we can add CM service to topology when auto-discovery is
+ turned on and CM service is selected in the descriptor
+ */
+ final ApiService cmService = new ApiService();
+ cmService.setName(CM_SERVICE_TYPE.toLowerCase(Locale.ROOT));
+ cmService.setType(CM_SERVICE_TYPE);
+ serviceList.addItemsItem(cmService);
+
for (ApiService service : serviceList.getItems()) {
String serviceName = service.getName();
log.discoveredService(serviceName, service.getType());
- ApiServiceConfig serviceConfig =
- getServiceConfig(servicesResourceApi, clusterName, serviceName);
+ ApiServiceConfig serviceConfig = null;
+ /* no reason to check service config for CM service */
+ if(!CM_SERVICE_TYPE.equals(service.getType())) {
+ serviceConfig =
+ getServiceConfig(servicesResourceApi, clusterName, serviceName);
+ }
ApiRoleList roleList = getRoles(rolesResourceApi, clusterName,
serviceName);
if (roleList != null) {
for (ApiRole role : roleList.getItems()) {
String roleName = role.getName();
log.discoveredServiceRole(roleName, role.getType());
- ApiConfigList roleConfig = getRoleConfig(rolesResourceApi,
clusterName, serviceName, roleName);
+ ApiConfigList roleConfig = null;
+ /* no reason to check role config for CM service */
+ if(!CM_SERVICE_TYPE.equals(service.getType())) {
+ roleConfig =
+ getRoleConfig(rolesResourceApi, clusterName, serviceName,
roleName);
+ }
List<ServiceModelGenerator> smgList =
serviceModelGenerators.get(service.getType());
if (smgList != null) {
@@ -280,7 +304,17 @@ public class ClouderaManagerServiceDiscovery implements
ServiceDiscovery {
String serviceName) {
ApiRoleList roles = null;
try {
- roles = rolesResourceApi.readRoles(clusterName, serviceName, "",
VIEW_SUMMARY);
+ /* Populate roles for CM Service since they are not discoverable */
+ if(CM_SERVICE_TYPE.equalsIgnoreCase(serviceName)) {
+ roles = new ApiRoleList();
+ final ApiRole cmRole = new ApiRole();
+ cmRole.setName(CM_ROLE_TYPE);
+ cmRole.setType(CM_ROLE_TYPE);
+ roles.addItemsItem(cmRole);
+ return roles;
+ } else {
+ roles = rolesResourceApi.readRoles(clusterName, serviceName, "",
VIEW_SUMMARY);
+ }
} catch (Exception e) {
log.failedToAccessServiceRoleConfigs(clusterName, e);
}
diff --git
a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java
new file mode 100644
index 0000000..0978362
--- /dev/null
+++
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.topology.discovery.cm.model.cm;
+
+import com.cloudera.api.swagger.client.ApiException;
+import com.cloudera.api.swagger.model.ApiConfigList;
+import com.cloudera.api.swagger.model.ApiRole;
+import com.cloudera.api.swagger.model.ApiService;
+import com.cloudera.api.swagger.model.ApiServiceConfig;
+import
org.apache.knox.gateway.topology.discovery.cm.ClouderaManagerServiceDiscovery;
+import org.apache.knox.gateway.topology.discovery.cm.ServiceModel;
+import
org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGenerator;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Locale;
+
+public class ClouderaManagerAPIServiceModelGenerator
+ extends AbstractServiceModelGenerator {
+
+ public static final String SERVICE = "CM-API";
+ public static final String SERVICE_TYPE =
ClouderaManagerServiceDiscovery.CM_SERVICE_TYPE;
+ public static final String ROLE_TYPE =
ClouderaManagerServiceDiscovery.CM_ROLE_TYPE;
+
+ @Override
+ public String getService() {
+ return SERVICE;
+ }
+
+ @Override
+ public String getServiceType() {
+ return SERVICE_TYPE;
+ }
+
+ @Override
+ public String getRoleType() {
+ return ROLE_TYPE;
+ }
+
+ @Override
+ public ServiceModel.Type getModelType() {
+ return ServiceModel.Type.API;
+ }
+
+ /**
+ * This method functions differently than others. This method inquires the
+ * discovery client and uses the CM url used by the driver (which was
+ * populated by the descriptor).
+ *
+ * @param service
+ * @param serviceConfig
+ * @param role
+ * @param roleConfig
+ * @return
+ * @throws ApiException
+ */
+ @Override
+ public ServiceModel generateService(ApiService service,
+ ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig)
+ throws ApiException {
+
+ final String basePath = getClient().getBasePath();
+ URI uri;
+ try {
+ uri = new URI(basePath);
+ } catch (URISyntaxException e) {
+ throw new ApiException(e);
+ }
+
+ final String serviceURL = getModelType() == ServiceModel.Type.API ?
+ String.format(Locale.getDefault(), "%s://%s:%s/api", uri.getScheme(),
+ uri.getHost(), uri.getPort()) :
+ String.format(Locale.getDefault(), "%s://%s:%s", uri.getScheme(),
+ uri.getHost(), uri.getPort());
+
+ return new ServiceModel(getModelType(), getService(), getServiceType(),
+ getRoleType(), serviceURL);
+ }
+}
diff --git
a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerUIServiceModelGenerator.java
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerUIServiceModelGenerator.java
new file mode 100644
index 0000000..7ba2b26
--- /dev/null
+++
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerUIServiceModelGenerator.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.topology.discovery.cm.model.cm;
+
+import org.apache.knox.gateway.topology.discovery.cm.ServiceModel;
+
+public class ClouderaManagerUIServiceModelGenerator
+ extends ClouderaManagerAPIServiceModelGenerator {
+
+ public static final String SERVICE = "CM-UI";
+
+ @Override
+ public String getService() {
+ return SERVICE;
+ }
+
+ @Override
+ public ServiceModel.Type getModelType() {
+ return ServiceModel.Type.UI;
+ }
+
+}
diff --git
a/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
b/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
index e6350ec..328160b 100644
---
a/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
+++
b/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
@@ -49,4 +49,6 @@
org.apache.knox.gateway.topology.discovery.cm.model.zeppelin.ZeppelinUIServiceMo
org.apache.knox.gateway.topology.discovery.cm.model.zeppelin.ZeppelinWSServiceModelGenerator
org.apache.knox.gateway.topology.discovery.cm.model.nifi.NifiServiceModelGenerator
org.apache.knox.gateway.topology.discovery.cm.model.nifi.NifiRegistryServiceModelGenerator
+org.apache.knox.gateway.topology.discovery.cm.model.cm.ClouderaManagerAPIServiceModelGenerator
+org.apache.knox.gateway.topology.discovery.cm.model.cm.ClouderaManagerUIServiceModelGenerator
diff --git
a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
index 853197d..d12dbc4 100644
---
a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
+++
b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
@@ -34,6 +34,7 @@ import
org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryConfig;
import
org.apache.knox.gateway.topology.discovery.cm.model.atlas.AtlasAPIServiceModelGenerator;
import
org.apache.knox.gateway.topology.discovery.cm.model.atlas.AtlasServiceModelGenerator;
+import
org.apache.knox.gateway.topology.discovery.cm.model.cm.ClouderaManagerUIServiceModelGenerator;
import
org.apache.knox.gateway.topology.discovery.cm.model.hbase.HBaseUIServiceModelGenerator;
import
org.apache.knox.gateway.topology.discovery.cm.model.hbase.WebHBaseServiceModelGenerator;
import
org.apache.knox.gateway.topology.discovery.cm.model.hdfs.NameNodeServiceModelGenerator;
@@ -69,6 +70,8 @@ import static org.junit.Assert.assertNotNull;
public class ClouderaManagerServiceDiscoveryTest {
+ private static final String DISCOVERY_URL = "http://localhost:1234";;
+
@Test
public void testJobTrackerServiceDiscovery() {
final String hostName = "resourcemanager-host-1";
@@ -562,6 +565,33 @@ public class ClouderaManagerServiceDiscoveryTest {
doTestNiFiRegistryDiscovery(true);
}
+ @Test
+ public void testCMDiscoveryUI() {
+ doTestCMDiscovery("CM-UI");
+ }
+
+ @Test
+ public void testCMDiscoveryAPI() {
+ doTestCMDiscovery("CM-API");
+ }
+
+ private void doTestCMDiscovery(final String serviceName) {
+ ServiceDiscovery.Cluster cluster = doTestDiscovery("somehost",
+ serviceName+"-1", ClouderaManagerUIServiceModelGenerator.SERVICE_TYPE,
+ serviceName+"-1",
+ ClouderaManagerUIServiceModelGenerator.ROLE_TYPE,
+ Collections.emptyMap(),
+ Collections.emptyMap());
+
+ List<String> urls = cluster.getServiceURLs(serviceName);
+ assertEquals(1, urls.size());
+ if("CM-UI".equals(serviceName)) {
+ assertEquals(DISCOVERY_URL, urls.get(0));
+ } else {
+ assertEquals(DISCOVERY_URL+"/api", urls.get(0));
+ }
+ }
+
private void doTestNiFiRegistryDiscovery(boolean sslEnabled) {
final String hostName = "nifi-registry-host";
final String port = "18080";
@@ -1071,7 +1101,7 @@ public class ClouderaManagerServiceDiscoveryTest {
private static ServiceDiscoveryConfig createMockDiscoveryConfig() {
- return createMockDiscoveryConfig("http://localhost:1234";, "itsme");
+ return createMockDiscoveryConfig(DISCOVERY_URL, "itsme");
}
private static ServiceDiscoveryConfig createMockDiscoveryConfig(String
address, String username) {
