Repository: ambari Updated Branches: refs/heads/trunk de5110ec4 -> c56844922
AMBARI-17492. Logsearch shows all the services not only those which are available in the stack (Miklos Gergely via oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c80ccd89 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c80ccd89 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c80ccd89 Branch: refs/heads/trunk Commit: c80ccd89c9d3087b215e2a2f6d9e72ddd4b0990f Parents: de5110e Author: Miklos Gergely <[email protected]> Authored: Wed Jul 6 12:13:16 2016 +0200 Committer: oleewere <[email protected]> Committed: Wed Jul 6 12:13:16 2016 +0200 ---------------------------------------------------------------------- .../ambari-logsearch-assembly/pom.xml | 3 +- .../ambari/server/agent/ExecutionCommand.java | 11 + .../AmbariCustomCommandExecutionHelper.java | 2 + .../AmbariManagementControllerImpl.java | 2 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 1 + .../0.5.0/package/scripts/setup_logsearch.py | 6 + .../templates/HadoopServiceConfig.json.j2 | 518 +++++++++++++++++++ .../stacks/2.4/LOGSEARCH/test_logsearch.py | 5 + .../test/python/stacks/2.4/configs/default.json | 4 + 9 files changed, 551 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-logsearch/ambari-logsearch-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/pom.xml b/ambari-logsearch/ambari-logsearch-assembly/pom.xml index 958232e..e0f6a09 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/pom.xml +++ b/ambari-logsearch/ambari-logsearch-assembly/pom.xml @@ -139,6 +139,7 @@ <exclude>classes/log4j.xml</exclude> <exclude>classes/logsearch.properties</exclude> <exclude>classes/user_pass.json</exclude> + <exclude>classes/HadoopServiceConfig.json</exclude> <exclude>solr_configsets/**</exclude> </excludes> </source> @@ -391,7 +392,7 @@ <group>root</group> </mapper> <excludes> - classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,solr_configsets/** + classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,classes/HadoopServiceConfig.json,solr_configsets/** </excludes> </data> <data> http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index f3f449d..9c262d9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -111,6 +111,9 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("localComponents") private Set<String> localComponents = new HashSet<String>(); + @SerializedName("availableServices") + private Set<String> availableServices = new HashSet<String>(); + public String getCommandId() { return commandId; } @@ -261,6 +264,14 @@ public class ExecutionCommand extends AgentCommand { this.localComponents = localComponents; } + public Set<String> getAvailableServices() { + return availableServices; + } + + public void setAvailableServices(Set<String> availableServices) { + this.availableServices = availableServices; + } + public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() { return configurationAttributes; } http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index fca66db..6750faf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -367,6 +367,8 @@ public class AmbariCustomCommandExecutionHelper { execCmd.setForceRefreshConfigTagsBeforeExecution(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION))); } + execCmd.setAvailableServices(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()).keySet()); + Map<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(CUSTOM_COMMAND, commandName); http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 5b4dae2..aaf69df 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2307,6 +2307,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } execCmd.setRoleParams(roleParams); + + execCmd.setAvailableServices(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()).keySet()); if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) { LOG.info("AmbariManagementControllerImpl.createHostAction: created ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, with cluster-env tags {}", http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index a89b3d6..165ac08 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -70,6 +70,7 @@ zookeeper_hosts_list.sort() # get comma separated list of zookeeper hosts from clusterHostInfo zookeeper_hosts = ",".join(zookeeper_hosts_list) cluster_name = str(config['clusterName']) +availableServices = config['availableServices'] # for now just pick first collector if 'metrics_collector_hosts' in config['clusterHostInfo']: http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index bfab139..57d6e8b 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -56,6 +56,12 @@ def setup_logsearch(): group=params.user_group ) + File(format("{logsearch_server_conf}/HadoopServiceConfig.json"), + content=Template("HadoopServiceConfig.json.j2"), + owner=params.logsearch_user, + group=params.user_group + ) + File(format("{logsearch_server_conf}/log4j.xml"), content=InlineTemplate(params.logsearch_app_log4j_content), owner=params.logsearch_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 new file mode 100644 index 0000000..64c81b5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 @@ -0,0 +1,518 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "service": { +{% if "ACCUMULO" in availableServices %} + "accumulo": { + "label": "Accumulo", + "components": [ + { + "name": "accumulo_gc" + }, + { + "name": "accumulo_master" + }, + { + "name": "accumulo_monitor" + }, + { + "name": "accumulo_tracer" + }, + { + "name": "accumulo_tserver" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "ATLAS" in availableServices %} + "atlas": { + "label": "Atlas", + "components": [ + { + "name": "atlas_app" + } + ], + "dependencies": [ + ] + }, +{% endif %} + "ambari": { + "label": "Ambari", + "components": [ + { + "name": "ambari_agent" + }, + { + "name": "ambari_server" + }, + { + "name": "ambari_alerts" + }, + { + "name": "ambari_audit" + }, + { + "name": "ambari_config_changes" + }, + { + "name": "ambari_eclipselink" + }, + { + "name": "ambari_server_check_database" + } + ], + "dependencies": [ + ] + }, +{% if "AMBARI_METRICS" in availableServices %} + "ams": { + "label": "AMS", + "components": [ + { + "name": "ams_hbase_master" + }, + { + "name": "ams_hbase_regionserver" + }, + { + "name": "ams_collector" + }, + { + "name": "ams_monitor" + }, + { + "name": "ams_grafana" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "FALCON" in availableServices %} + "falcon": { + "label": "Falcon", + "components": [ + { + "name": "falcon_app" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "FLUME" in availableServices %} + "flume": { + "label": "Flume", + "components": [ + { + "name": "flume_handler" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "HBASE" in availableServices %} + "hbase": { + "label": "HBase", + "components": [ + { + "name": "hbase_master" + }, + { + "name": "hbase_regionserver" + }, + { + "name": "hbase_phoenix_server" + } + ], + "dependencies": [ + { + "service": "hdfs", + "components": [ + "hdfs_namenode" + ] + } + ] + }, +{% endif %} +{% if "HDFS" in availableServices %} + "hdfs": { + "label": "HDFS", + "components": [ + { + "name": "hdfs_datanode" + }, + { + "name": "hdfs_namenode" + }, + { + "name": "hdfs_journalnode" + }, + { + "name": "hdfs_secondarynamenode" + }, + { + "name": "hdfs_zkfc" + }, + { + "name": "hdfs_nfs3" + }, + { + "name": "hdfs_audit", + "rowtype": "audit" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "HIVE" in availableServices %} + "hive": { + "label": "Hive", + "components": [ + { + "name": "hive_hiveserver2" + }, + { + "name": "hive_metastore" + }, + { + "name": "webhcat_server" + } + ], + "dependencies": [ + { + "service": "hdfs", + "components": [ + "hdfs_namenode" + ] + } + ] + }, +{% endif %} +{% if "KAFKA" in availableServices %} + "kafka": { + "label": "Kafka", + "components": [ + { + "name": "kafka_controller" + }, + { + "name": "kafka_request" + }, + { + "name": "kafka_logcleaner" + }, + { + "name": "kafka_server" + }, + { + "name": "kafka_statechange" + } + ], + "dependencies": [ + { + "service": "zookeeper", + "components": [ + "zookeeper" + ] + } + ] + }, +{% endif %} +{% if "KNOX" in availableServices %} + "knox": { + "label": "Knox", + "components": [ + { + "name": "knox_gateway" + }, + { + "name": "knox_cli" + }, + { + "name": "knox_ldap" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "NIFI" in availableServices %} + "nifi": { + "label": "NiFi", + "components": [ + { + "name": "nifi_app" + }, + { + "name": "nifi_bootstrap" + }, + { + "name": "nifi_setup" + }, + { + "name": "nifi_user" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "MAPREDUCE2" in availableServices %} + "mapred": { + "label": "MapReduce", + "components": [ + { + "name": "mapred_historyserver" + } + ], + "dependencies": [ + ] + }, +{% endif %} + "logsearch": { + "label": "Logsearch", + "components": [ + { + "name": "logsearch_app" + }, + { + "name": "logsearch_feeder" + }, + { + "name": "logsearch_perf" + }, + { + "name": "logsearch_solr" + } + ], + "dependencies": [ + ] + }, +{% if "RANGER" in availableServices %} + "ranger": { + "label": "Ranger", + "components": [ + { + "name": "ranger_admin" + }, + { + "name": "ranger_dbpatch" + }, + { + "name": "ranger_kms" + }, + { + "name": "ranger_usersync" + } + ], + "dependencies": [ + { + "service": "hdfs", + "required": "optional", + "components": [ + "hdfs_namenode" + ] + }, + { + "service": "hbase", + "required": "optional", + "components": [ + "hbase_master", + "hbase_regionserver" + ] + }, + { + "service": "hive", + "required": "optional", + "components": [ + "hive_hiveserver2" + ] + }, + { + "service": "kafka", + "required": "optional", + "components": [ + "kafka_ranger" + ] + }, + { + "service": "knox", + "required": "optional", + "components": [ + "knox_gateway" + ] + }, + { + "service": "storm", + "required": "optional", + "components": [ + "storm_supervisor" + ] + }, + { + "service": "yarn", + "required": "optional", + "components": [ + "yarn_resourcemanager" + ] + } + ] + }, +{% endif %} +{% if "OOZIE" in availableServices %} + "oozie": { + "label": "Oozie", + "components": [ + { + "name": "oozie_app" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "SMARTSENSE" in availableServices %} + "hst": { + "label": "SmartSense", + "components": [ + { + "name": "hst_server" + }, + { + "name": "hst_agent" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "SPARK" in availableServices %} + "spark": { + "label": "Spark", + "components": [ + { + "name": "spark_jobhistory_server" + }, + { + "name": "spark_thriftserver" + }, + { + "name": "livy_server" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "SPARK2" in availableServices %} + "spark2": { + "label": "Spark 2", + "components": [ + { + "name": "spark2_jobhistory_server" + }, + { + "name": "spark2_thriftserver" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "STORM" in availableServices %} + "storm": { + "label": "Storm", + "components": [ + { + "name": "storm_drpc" + }, + { + "name": "storm_logviewer" + }, + { + "name": "storm_nimbus" + }, + { + "name": "storm_supervisor" + }, + { + "name": "storm_ui" + }, + { + "name": "storm_worker" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "YARN" in availableServices %} + "yarn": { + "label": "YARN", + "components": [ + { + "name": "yarn_nodemanager" + }, + { + "name": "yarn_resourcemanager" + }, + { + "name": "yarn_timelineserver" + }, + { + "name": "yarn_historyserver" + }, + { + "name": "yarn_jobsummary" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "ZEPPELIN" in availableServices %} + "zeppelin": { + "label": "Zeppelin", + "components": [ + { + "name": "zeppelin" + } + ], + "dependencies": [ + ] + }, +{% endif %} +{% if "ZOOKEEPER" in availableServices %} + "zookeeper": { + "label": "ZooKeeper", + "components": [ + { + "name": "zookeeper" + } + ], + "dependencies": [ + ] + } +{% endif %} + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py index 48e071d..cd45027 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -78,6 +78,11 @@ class TestLogSearch(RMFTestCase): group='hadoop', content = Template('logsearch.properties.j2') ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/HadoopServiceConfig.json', + owner = 'logsearch', + group='hadoop', + content = Template('HadoopServiceConfig.json.j2') + ) self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/log4j.xml', owner = 'logsearch', group='hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index 233a733..462ee56 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -32,6 +32,10 @@ }, "taskId": 152, "public_hostname": "c6401.ambari.apache.org", + "availableServices": [ + "ACCUMULO", + "ATLAS" + ], "configurations": { "slider-client": { "slider.yarn.queue": "default"
