Repository: ambari Updated Branches: refs/heads/branch-2.5 2ea4a0328 -> d57dc455b
AMBARI-19870. Add Superset as a UI for Druid in HDP (Nishant Bangarwa via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d57dc455 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d57dc455 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d57dc455 Branch: refs/heads/branch-2.5 Commit: d57dc455b73829ab7c5e72dcc34920dbc308da87 Parents: 2ea4a03 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Thu Feb 9 12:32:17 2017 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Feb 9 12:33:14 2017 -0800 ---------------------------------------------------------------------- .../DRUID/0.9.2/configuration/druid-common.xml | 10 +- .../0.9.2/configuration/druid-superset-env.xml | 115 ++++++++++++ .../0.9.2/configuration/druid-superset.xml | 178 +++++++++++++++++++ .../common-services/DRUID/0.9.2/metainfo.xml | 26 ++- .../DRUID/0.9.2/package/scripts/params.py | 45 ++++- .../0.9.2/package/scripts/status_params.py | 2 + .../DRUID/0.9.2/package/scripts/superset.py | 153 ++++++++++++++++ .../DRUID/0.9.2/package/templates/superset.sh | 95 ++++++++++ .../DRUID/0.9.2/quicklinks/quicklinks.json | 13 ++ .../DRUID/0.9.2/role_command_order.json | 18 ++ .../DRUID/0.9.2/themes/theme.json | 84 ++++++++- .../stacks/HDP/2.6/role_command_order.json | 19 -- .../stacks/HDP/2.6/services/DRUID/kerberos.json | 33 ++++ .../stacks/HDP/2.6/services/stack_advisor.py | 11 ++ .../test/python/stacks/2.6/configs/default.json | 3 + 15 files changed, 774 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml index a494750..d3b53cd 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml @@ -132,20 +132,20 @@ <property> <name>database_name</name> <value>druid</value> - <display-name>Metadata storage database name</display-name> - <description>Metadata storage database name</description> + <display-name>Druid Metadata storage database name</display-name> + <description>Druid Metadata storage database name</description> <on-ambari-upgrade add="false"/> </property> <property> <name>metastore_hostname</name> <value>localhost</value> - <display-name>Metadata storage hostname name</display-name> - <description>Metadata storage hostname name</description> + <display-name>Metadata storage hostname</display-name> + <description>Metadata storage hostname</description> <on-ambari-upgrade add="false"/> </property> <property require-input="true"> <name>druid.metadata.storage.type</name> - <display-name>Metadata storage type</display-name> + <display-name>Druid Metadata storage type</display-name> <value>derby</value> <value-attributes> <overridable>false</overridable> http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml new file mode 100644 index 0000000..61421c0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml @@ -0,0 +1,115 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<configuration> + <property> + <name>superset_log_dir</name> + <value>/var/log/superset</value> + <display-name>Superset log dir</display-name> + <description>Superset log directory.</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>superset_pid_dir</name> + <value>/var/run/superset</value> + <display-name>Superset pid dir</display-name> + <description>Superset pid directory.</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>superset_admin_user</name> + <value>admin</value> + <display-name>Superset Admin Username</display-name> + <description> + Superset Username. This value cannot be modified by Ambari + except on initial install. Please make sure the username change in + Superset is reflected in Ambari. + </description> + <value-attributes> + <type>db_user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property require-input="true"> + <name>superset_admin_password</name> + <value/> + <property-type>PASSWORD</property-type> + <display-name>Superset Admin Password</display-name> + <description> + Superset password. This value cannot be modified by Ambari + except on initial install. Please make sure the password change in + Superset is reflected back in Ambari. + </description> + <value-attributes> + <overridable>false</overridable> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>superset_admin_firstname</name> + <value></value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>superset_admin_lastname</name> + <value></value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>superset_admin_email</name> + <value></value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>superset-env template</display-name> + <value> + # Set environment variables here. + + # Superset Home Dir + export SUPERSET_CONFIG_DIR={{superset_config_dir}} + + # Superset Log Dir + export SUPERSET_LOG_DIR={{superset_log_dir}} + + # Superset Log Dir + export SUPERSET_BIN_DIR={{superset_bin_dir}} + + # Superset PID Dir + export SUPERSET_PID_DIR={{superset_pid_dir}} + + # Add superset_config.py to PYTHONPATH + export PYTHONPATH=$PYTHONPATH:{{superset_config_dir}} + + export SUPERSET_WEBSERVER_PORT={{superset_webserver_port}} + export SUPERSET_WEBSERVER_ADDRESS={{superset_webserver_address}} + export SUPERSET_TIMEOUT={{superset_timeout}} + export SUPERSET_WORKERS={{superset_workers}} + </value> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml new file mode 100644 index 0000000..2fff10a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml @@ -0,0 +1,178 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>ROW_LIMIT</name> + <value>5000</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <description></description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_WORKERS</name> + <value>4</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <description></description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_WEBSERVER_PORT</name> + <value>9088</value> + <description></description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_WEBSERVER_ADDRESS</name> + <value>0.0.0.0</value> + <description></description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_TIMEOUT</name> + <value>60</value> + <description></description> + <on-ambari-upgrade add="false"/> + </property> + <property require-input="true"> + <name>SUPERSET_DATABASE_TYPE</name> + <display-name>Superset Database type</display-name> + <value>sqlite</value> + <value-attributes> + <overridable>false</overridable> + <type>value-list</type> + <entries> + <entry> + <value>mysql</value> + <label>MYSQL</label> + </entry> + <entry> + <value>sqlite</value> + <label>SQLITE</label> + </entry> + <entry> + <value>postgresql</value> + <label>POSTGRESQL</label> + </entry> + </entries> + </value-attributes> + <description>Type of the Database. Note that sqlite db will be installed on the node where superset is installed. + Use mysql or postgred when installing superset on multiple nodes. + mysql installed by ambari is only for development and not suitable for production use cases due to it being not HA. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_DATABASE_NAME</name> + <value>superset</value> + <display-name>Superset Database name</display-name> + <description>Superset Database name</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_DATABASE_PASSWORD</name> + <value></value> + <property-type>PASSWORD</property-type> + <display-name>Superset Database password</display-name> + <description>Password for the database.</description> + <value-attributes> + <type>password</type> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_DATABASE_USER</name> + <value>superset</value> + <display-name>Superset Database user</display-name> + <description>Superset Database user</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_DATABASE_PORT</name> + <value></value> + <display-name>Superset Database port</display-name> + <description>Superset Database port</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>superset</type> + <name>SUPERSET_DATABASE_TYPE</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_DATABASE_NAME</name> + <value>superset</value> + <display-name>Superset Database name</display-name> + <description>Superset Database name</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SUPERSET_DATABASE_HOSTNAME</name> + <value>localhost</value> + <display-name>Database hostname</display-name> + <description>Database hostname</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>WTF_CSRF_ENABLED</name> + <value>True</value> + <description></description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ENABLE_PROXY_FIX</name> + <value>True</value> + <description>When using Superset beind a Proxy server set this to True.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>MAPBOX_API_KEY</name> + <value></value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <description>API key to enable Mapbox visualizations</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>SECRET_KEY</name> + <value></value> + <value-attributes> + <type>password</type> + </value-attributes> + <description>Secret Key used to encrypt user passwords. A Long Random String is recommended.</description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml index c837bbb..c0859f2 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml @@ -158,10 +158,24 @@ <scriptType>PYTHON</scriptType> <timeout>600</timeout> </commandScript> + <configuration-dependencies> + <config-type>druid-router</config-type> + </configuration-dependencies> + </component> + <component> + <name>DRUID_SUPERSET</name> + <displayName>Druid Superset</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/superset.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <configuration-dependencies> + <config-type>druid-superset</config-type> + </configuration-dependencies> </component> - <configuration-dependencies> - <config-type>druid-router</config-type> - </configuration-dependencies> </components> <themes> @@ -178,6 +192,9 @@ <package> <name>druid_${stack_version}</name> </package> + <package> + <name>superset_${stack_version}</name> + </package> </packages> </osSpecific> <osSpecific> @@ -186,6 +203,9 @@ <package> <name>druid-${stack_version}</name> </package> + <package> + <name>superset-${stack_version}</name> + </package> </packages> </osSpecific> </osSpecifics> http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py index aed4043..61f35b2 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py @@ -149,7 +149,6 @@ HdfsResource = functools.partial( dfs_type=dfs_type ) - # Ambari Metrics metric_emitter_type = "noop" metric_collector_host = "" @@ -184,3 +183,47 @@ if has_metric_collector: metric_collector_protocol = 'http' pass +superset_home_dir = format("{stack_root}/current/druid-superset") +superset_bin_dir = format("{superset_home_dir}/bin") +superset_log_dir = default("/configurations/druid-superset-env/superset_log_dir", '/var/log/superset') +superset_pid_dir = status_params.superset_pid_dir +superset_config_dir = '/etc/superset/conf' +superset_admin_user = config['configurations']['druid-superset-env']['superset_admin_user'] +superset_admin_password = config['configurations']['druid-superset-env']['superset_admin_password'] +superset_admin_firstname = config['configurations']['druid-superset-env']['superset_admin_firstname'] +superset_admin_lastname = config['configurations']['druid-superset-env']['superset_admin_lastname'] +superset_admin_email = config['configurations']['druid-superset-env']['superset_admin_email'] +superset_env_sh_template = config['configurations']['druid-superset-env']['content'] +superset_protocol = "http" +superset_webserver_address=config['configurations']['druid-superset']['SUPERSET_WEBSERVER_ADDRESS'] +superset_webserver_port = config['configurations']['druid-superset']['SUPERSET_WEBSERVER_PORT'] +superset_timeout = config['configurations']['druid-superset']['SUPERSET_TIMEOUT'] +superset_workers = config['configurations']['druid-superset']['SUPERSET_WORKERS'] +superset_hosts = default('/clusterHostInfo/superset_hosts', None) + +# superset database configs +superset_db_type = config['configurations']['druid-superset']['SUPERSET_DATABASE_TYPE'] +superset_db_name = config['configurations']['druid-superset']['SUPERSET_DATABASE_NAME'] +superset_db_password = config['configurations']['druid-superset']['SUPERSET_DATABASE_PASSWORD'] +superset_db_user = config['configurations']['druid-superset']['SUPERSET_DATABASE_USER'] +superset_db_port = config['configurations']['druid-superset']['SUPERSET_DATABASE_PORT'] +superset_db_host = config['configurations']['druid-superset']['SUPERSET_DATABASE_HOSTNAME'] + +superset_db_uri = None +if superset_db_type == "sqlite": + superset_db_uri = format("sqlite:///{superset_config_dir}/{superset_db_name}.db") +elif superset_db_type == "postgresql": + superset_db_uri = format("postgresql+pygresql://{superset_db_user}:{superset_db_password}@{superset_db_host}:{superset_db_port}/{superset_db_name}") +elif superset_db_type == "mysql": + superset_db_uri = format("mysql+pymysql://{superset_db_user}:{superset_db_password}@{superset_db_host}:{superset_db_port}/{superset_db_name}") + +druid_coordinator_hosts = default("/clusterHostInfo/druid_coordinator_hosts", []) +druid_coordinator_host = "" +if not len(druid_coordinator_hosts) == 0: + druid_coordinator_host = druid_coordinator_hosts[0] +druid_router_hosts = default("/clusterHostInfo/druid_coordinator_hosts", []) +druid_router_host = "" +if not len(druid_router_hosts) == 0: + druid_router_host = druid_router_hosts[0] +druid_coordinator_port = config['configurations']['druid-coordinator']['druid.port'] +druid_router_port = config['configurations']['druid-router']['druid.port'] http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py index ee1d61c..d48ff83 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py @@ -22,3 +22,5 @@ from resource_management.libraries.script.script import Script config = Script.get_config() druid_pid_dir = config['configurations']['druid-env']['druid_pid_dir'] +superset_pid_dir = config['configurations']['druid-superset-env']['superset_pid_dir'] + http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py new file mode 100644 index 0000000..6a56e25 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py @@ -0,0 +1,153 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os, time +from resource_management.core.resources.system import Directory +from resource_management import Script +from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.core.resources.system import Execute +from resource_management.core.source import Template +from resource_management.libraries.functions.show_logs import show_logs +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.core.resources import File + +class Superset(Script): + + def get_component_name(self): + return format("druid-superset") + + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + Directory( + [params.superset_pid_dir, params.superset_log_dir, params.superset_config_dir, params.superset_home_dir], + mode=0755, + cd_access='a', + owner=params.druid_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True + ) + + File(format("{params.superset_config_dir}/superset-env.sh"), + mode=0755, + owner=params.druid_user, + group=params.user_group, + content=InlineTemplate(params.superset_env_sh_template) + ) + + File(os.path.join(params.superset_bin_dir, 'superset.sh'), + owner=params.druid_user, + group=params.user_group, + mode=0755, + content=Template("superset.sh") + ) + superset_config = mutable_config_dict(params.config["configurations"]["druid-superset"]) + + if params.superset_db_uri: + superset_config["SQLALCHEMY_DATABASE_URI"] = params.superset_db_uri + + PropertiesFile("superset_config.py", + dir=params.superset_config_dir, + properties=quote_string_values(superset_config), + owner=params.druid_user, + group=params.user_group + ) + + # Initialize DB and create admin user. + Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset db upgrade"), + user=params.druid_user) + Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/fabmanager create-admin --app superset --username {params.superset_admin_user} --password {params.superset_admin_password} --firstname {params.superset_admin_firstname} --lastname {params.superset_admin_lastname} --email {params.superset_admin_email}"), + user=params.druid_user) + Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset init"), + user=params.druid_user) + + # Configure Druid Cluster in superset DB + Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset configure_druid_cluster --name druid-ambari --coordinator-host {params.druid_coordinator_host} --coordinator-port {params.druid_coordinator_port} --broker-host {params.druid_router_host} --broker-port {params.druid_router_port} --coordinator-endpoint druid/coordinator/v1/metadata --broker-endpoint druid/v2"), + user=params.druid_user) + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + daemon_cmd = self.get_daemon_cmd(params, "start") + try: + Execute(daemon_cmd, + user=params.druid_user + ) + except: + show_logs(params.druid_log_dir, params.druid_user) + raise + + + def stop(self, env): + import params + env.set_params(params) + + daemon_cmd = self.get_daemon_cmd(params, "stop") + try: + Execute(daemon_cmd, + user=params.druid_user + ) + except: + show_logs(params.druid_log_dir, params.druid_user) + raise + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = status_params.superset_pid_dir + '/superset.pid' + check_process_status(pid_file) + + def get_log_folder(self): + import params + return params.druid_log_dir + + def get_user(self): + import params + return params.druid_user + + def get_daemon_cmd(self, params=None, command=None): + return format('source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset.sh {command}') + +def mutable_config_dict(config): + rv = {} + for key, value in config.iteritems(): + rv[key] = value + return rv + +def quote_string_values(config): + rv = {} + for key, value in config.iteritems(): + rv[key] = quote_string_value(value) + return rv + +def quote_string_value(value): + if value.lower() == "true" or value.lower() == "false" or value.isdigit(): + return value + else: + return "'{}'".format(value) + + + +if __name__ == "__main__": + Superset().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh new file mode 100644 index 0000000..3e327a6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +## Runs superset as a daemon +## Environment Variables used by this script - +## SUPERSET_CONFIG_DIR - directory having druid config files +## SUPERSET_LOG_DIR - directory used to store superset logs +## SUPERSET_PID_DIR - directory used to store pid file + +usage="Usage: superset.sh (start|stop|status)" + +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +command=$1 + +CONF_DIR="${SUPERSET_CONFIG_DIR:=/etc/superset/conf}" +LOG_DIR="${SUPERSET_LOG_DIR:=/var/log/superset}" +PID_DIR="${SUPERSET_PID_DIR:=/var/run/superset}" +TIMEOUT="${SUPERSET_TIMEOUT:=60}" +WEBSERVER_ADDRESS="${SUPERSET_WEBSERVER_ADDRESS:=0.0.0.0}" +WEBSERVER_PORT="${SUPERSET_WEBSERVER_PORT:=9088}" +WORKERS="${SUPERSET_WORKERS:=4}" +BIN_DIR="${SUPERSET_BIN_DIR}" + +pid=$PID_DIR/superset.pid + +case $command in + (start) + + if [ -f $pid ]; then + if kill -0 `cat $pid| head -n 1` > /dev/null 2>&1; then + echo Superset node running as process `cat $pid | head -n 1`. Stop it first. + exit 1 + fi + fi + + $BIN_DIR/gunicorn -D --workers $WORKERS -p $pid --log-file $LOG_DIR/superset.log -t $TIMEOUT -b $WEBSERVER_ADDRESS:$WEBSERVER_PORT --limit-request-line 0 --limit-request-field_size 0 superset:app + + echo "Started Superset" + ;; + + (stop) + + if [ -f $pid ]; then + TARGET_PID=`cat $pid | head -n 1` + if kill -0 $TARGET_PID > /dev/null 2>&1; then + echo Stopping process `cat $pid | head -n 1`... + kill $TARGET_PID + else + echo No superset node to stop + fi + rm -f $pid + else + echo No superset node to stop + fi + ;; + + (status) + if [ -f $pid ]; then + if kill -0 `cat $pid | head -n 1` > /dev/null 2>&1; then + echo RUNNING + exit 0 + else + echo STOPPED + fi + else + echo STOPPED + fi + ;; + + (*) + echo $usage + exit 1 + ;; +esac \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json index c68b9b9..16f5d5c 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json @@ -31,6 +31,19 @@ "regex": "^(\\d+)$", "site": "druid-overlord" } + }, + { + "name": "superset", + "label": "Superset", + "component_name": "DRUID_SUPERSET", + "requires_user_name": "false", + "url": "%@://%@:%@", + "port": { + "http_property": "SUPERSET_WEBSERVER_PORT", + "http_default_port": "9088", + "regex": "^(\\d+)$", + "site": "druid-superset" + } } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json new file mode 100644 index 0000000..8f65c2e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json @@ -0,0 +1,18 @@ +{ + "general_deps" : { + "_comment" : "dependencies for Druid", + "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], + "DRUID_SUPERSET-START" : ["DRUID_COORDINATOR-START", "DRUID_BROKER-START"], + "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"], + "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"], + "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"], + "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"], + "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"], + "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START", "DRUID_SUPERSET-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json index f494594..151478b 100644 --- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json @@ -10,7 +10,7 @@ "name": "metadata_storage", "display-name": "META DATA STORAGE CONFIG", "layout": { - "tab-columns": "1", + "tab-columns": "2", "tab-rows": "1", "sections": [ { @@ -19,17 +19,25 @@ "row-index": "0", "column-index": "0", "row-span": "2", - "column-span": "1", + "column-span": "2", "section-columns": "1", "section-rows": "1", "subsections": [ { "name": "subsection-metadata-storage-row1-col1", - "display-name": "META DATA STORAGE", + "display-name": "DRUID META DATA STORAGE", "row-index": "0", "column-index": "0", "row-span": "1", "column-span": "1" + }, + { + "name": "subsection-metadata-storage-row1-col2", + "display-name": "SUPERSET META DATA STORAGE", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" } ] } @@ -69,6 +77,34 @@ { "config": "druid-common/druid.metadata.storage.connector.connectURI", "subsection-name": "subsection-metadata-storage-row1-col1" + }, + { + "config": "druid-superset/SUPERSET_DATABASE_NAME", + "subsection-name": "subsection-metadata-storage-row1-col2" + }, + { + "config": "druid-superset/SUPERSET_DATABASE_TYPE", + "subsection-name": "subsection-metadata-storage-row1-col2" + }, + { + "config": "druid-superset/SUPERSET_DATABASE_USER", + "subsection-name": "subsection-metadata-storage-row1-col2" + }, + { + "config": "druid-superset/SUPERSET_DATABASE_PASSWORD", + "subsection-name": "subsection-metadata-storage-row1-col2" + }, + { + "config": "druid-superset/SUPERSET_DATABASE_HOSTNAME", + "subsection-name": "subsection-metadata-storage-row1-col2" + }, + { + "config": "druid-superset/SUPERSET_DATABASE_PORT", + "subsection-name": "subsection-metadata-storage-row1-col2" + }, + { + "config": "druid-superset/SECRET_KEY", + "subsection-name": "subsection-metadata-storage-row1-col2" } ] }, @@ -114,6 +150,48 @@ "widget": { "type": "text-field" } + }, + { + "config": "druid-superset/SUPERSET_DATABASE_NAME", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-superset/SUPERSET_DATABASE_TYPE", + "widget": { + "type": "combo" + } + }, + { + "config": "druid-superset/SUPERSET_DATABASE_USER", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-superset/SUPERSET_DATABASE_PASSWORD", + "widget": { + "type": "password" + } + }, + { + "config": "druid-superset/SUPERSET_DATABASE_HOSTNAME", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-superset/SUPERSET_DATABASE_PORT", + "widget": { + "type": "text-field" + } + }, + { + "config": "druid-superset/SECRET_KEY", + "widget": { + "type": "password" + } } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json deleted file mode 100644 index 66b6ed2..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "_comment" : "Record format:", - "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", - "general_deps" : { - "_comment" : "dependencies for all cases", - "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"], - "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"], - "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"], - "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"], - "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"], - "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"], - "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START"] - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json index 251975b..0ba5071 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json @@ -34,6 +34,27 @@ } }, { + "name": "superset", + "principal": { + "value": "${druid-env/druid_user}@${realm}", + "type": "user", + "configuration": "druid-superset/KERBEROS_PRINCIPAL", + "local_username": "${druid-env/druid_user}" + }, + "keytab": { + "file": "${keytab_dir}/superset.headless.keytab", + "owner": { + "name": "${druid-env/druid_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + }, + "configuration": "druid-superset/KERBEROS_KEYTAB" + } + }, + { "name": "/smokeuser" } ], @@ -77,6 +98,14 @@ "name": "/druid" } ] + }, + { + "name": "DRUID_SUPERSET", + "identities": [ + { + "name": "/druid" + } + ] } ], "configurations": [ @@ -84,6 +113,10 @@ "druid-common": { "druid.hadoop.security.spnego.excludedPaths": "[\"/status\"]", "druid.security.extensions.loadList" : "[\"druid-kerberos\"]" + }, + "druid-superset": { + "ENABLE_KERBEROS_AUTHENTICATION" : "True", + "KERBEROS_REINIT_TIME_SEC" : 3600 } } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 957d625..e6ebd6f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -127,6 +127,17 @@ class HDP26StackAdvisor(HDP25StackAdvisor): putComponentProperty('druid.processing.numThreads', processingThreads) putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30) + # superset is in list of services to be installed + if 'druid-superset' in services['configurations']: + # Recommendations for Superset + superset_database_type = services['configurations']["druid-superset"]["properties"]["SUPERSET_DATABASE_TYPE"] + putSupersetProperty = self.putProperty(configurations, "druid-superset", services) + + if superset_database_type == "mysql": + putSupersetProperty("SUPERSET_DATABASE_PORT", "3306") + elif superset_database_type == "postgresql": + putSupersetProperty("SUPERSET_DATABASE_PORT", "5432") + def getMetadataConnectionString(self, database_type): driverDict = { 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true', http://git-wip-us.apache.org/repos/asf/ambari/blob/d57dc455/ambari-server/src/test/python/stacks/2.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/configs/default.json b/ambari-server/src/test/python/stacks/2.6/configs/default.json index 4d9f98c..a1f89ce 100644 --- a/ambari-server/src/test/python/stacks/2.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.6/configs/default.json @@ -459,6 +459,9 @@ }, "druid-logrotate" : { "content" : "<![CDATA[\n {{druid_log_dir}}/*.log {\n copytruncate\n rotate 7\n daily\n nocompress\n missingok\n notifempty\n create 660 druid users\n dateext\n dateformat -%Y-%m-%d-%s\n }\n ]]>\n" + }, + "druid-superset" : { + "SUPERSET_DATABASE_TYPE" : "sqllite" } }, "configuration_attributes": {