This is an automated email from the ASF dual-hosted git repository. jialiang pushed a commit to branch eee in repository https://gitbox.apache.org/repos/asf/ambari.git
commit e30687f875291c8548926091d0f1645e742060b6 Author: jialiang <[email protected]> AuthorDate: Sat Sep 7 11:02:14 2024 +0800 add dinky --- .../stacks/BIGTOP/3.2.0/services/DINKY/alerts.json | 32 ++++ .../configuration/dinky-application-server.xml | 141 +++++++++++++++ .../services/DINKY/configuration/dinky-env.xml | 74 ++++++++ .../BIGTOP/3.2.0/services/DINKY/metainfo.xml | 99 +++++++++++ .../services/DINKY/package/scripts/dinky_env.py | 70 ++++++++ .../services/DINKY/package/scripts/dinky_server.py | 89 ++++++++++ .../3.2.0/services/DINKY/package/scripts/params.py | 108 ++++++++++++ .../DINKY/package/scripts/service_check.py | 32 ++++ .../package/templates/application-mysql.yml.j2 | 24 +++ .../package/templates/application-pgsql.yml.j2 | 23 +++ .../DINKY/package/templates/application.yml.j2 | 153 ++++++++++++++++ .../services/DINKY/package/templates/auto.sh.j2 | 148 ++++++++++++++++ .../services/DINKY/quicklinks/quicklinks.json | 26 +++ .../BIGTOP/3.2.0/services/DINKY/service_advisor.py | 196 +++++++++++++++++++++ 14 files changed, 1215 insertions(+) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/alerts.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/alerts.json new file mode 100644 index 0000000000..7ddefb6d23 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/alerts.json @@ -0,0 +1,32 @@ +{ + "DINKY": { + "service": [], + "DINKY_SERVER": [ + { + "name": "dinky_server_port_check", + "label": "dinky_server_port_check", + "description": "dinky_server_port_check.", + "interval": 10, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{dinky-application-server/server.port}}", + "default_port": 8888, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ] + } +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/configuration/dinky-application-server.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/configuration/dinky-application-server.xml new file mode 100644 index 0000000000..79d78049bd --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/configuration/dinky-application-server.xml @@ -0,0 +1,141 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--> +<configuration> + <!-- server port --> + <property require-input="true"> + <name>server.port</name> + <value>8888</value> + <property-type> + <type>PORT</type> + <default-port>8888</default-port> + </property-type> + <on-ambari-upgrade add="true"/> + </property> + + <!-- flink big version --> + <property require-input="true"> + <name>flink.big.version</name> + <value>1.17</value> + <description> + en: The version of flink big version such as: 1.18, 1.17, 1.16, 1.15, 1.14 + Note: The version of flink big version must be consistent with the version of flink big version installed on + the cluster. + | + zh: Flink大版本号，如：1.18, 1.17, 1.16, 1.15, 1.14 + 注意：Flink大版本号必须与集群上安装的Flink大版本号保持一致。 + </description> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>1.14</value> + </entry> + <entry> + <value>1.15</value> + </entry> + <entry> + <value>1.16</value> + </entry> + <entry> + <value>1.17</value> + </entry> + </entries> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <!-- 数据库类型 --> + <property require-input="true"> + <name>spring.profiles.active</name> + <value>mysql</value> + <description> + en: The type of database to use. Valid values are mysql, postgresql , you must input the mysql/postgresql + information.| zh: 数据库类型，可选值为mysql, postgresql,必须输入mysql/postgresql的信息。 + </description> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>mysql</value> + <description> + en: mysql database | zh: mysql数据库 + </description> + </entry> + <entry> + <value>pgsql</value> + <description> + en: postgresql database | zh: postgresql数据库 + </description> + </entry> + </entries> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <!-- 数据库主机 --> + <property require-input="true"> + <name>spring.datasource.database.host</name> + <value>127.0.0.1</value> + <description> + en: DataBase host ip/hostname | zh: 数据库主机ip/主机名 + </description> + <on-ambari-upgrade add="true"/> + </property> + + <!-- 数据库端口 --> + <property require-input="true"> + <name>spring.datasource.database.port</name> + <value>3306</value> + <description> + en: DataBase port | zh: 数据库端口 + </description> + <property-type>PORT</property-type> + <on-ambari-upgrade add="true"/> + </property> + + <!-- 数据库名称 --> + <property require-input="true"> + <name>spring.datasource.database.name</name> + <value>dinky</value> + <description> + en: DataBase Name | zh: 数据库名称 + </description> + <on-ambari-upgrade add="true"/> + </property> + + <!-- 数据库用户名 --> + <property require-input="true"> + <name>spring.datasource.database.username</name> + <value>dinky</value> + <description> + en: DataBase UserName | zh: 数据库用户名 + </description> + <on-ambari-upgrade add="true"/> + </property> + + <!-- 数据库密码 --> + <property require-input="true"> + <name>spring.datasource.database.password</name> + <value>dinky1234!2</value> + <description> + en: DataBase Password | zh: 数据库密码 + </description> + <property-type>PASSWORD</property-type> + <on-ambari-upgrade add="true"/> + </property> +</configuration> \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/configuration/dinky-env.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/configuration/dinky-env.xml new file mode 100755 index 0000000000..4279e8d9bc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/configuration/dinky-env.xml @@ -0,0 +1,74 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>dinky_user</name> + <display-name>Dinky User</display-name> + <value>dinky</value> + <property-type>USER</property-type> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + <user-groups> + <property> + <type>cluster-env</type> + <name>user_group</name> + </property> + <property> + <type>dinky-env</type> + <name>dinky_group</name> + </property> + </user-groups> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dinky_group</name> + <display-name>Dinky Group</display-name> + <value>hadoop</value> + <property-type>GROUP</property-type> + <description>dinky group</description> + <value-attributes> + <type>user</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dinky_log_dir</name> + <display-name>Dinky Log directory</display-name> + <value>/var/log/dinky</value> + <description>Dinky Log Dir</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dinky_pid_dir</name> + <display-name>Dinky PID directory</display-name> + <value>/var/run/dinky</value> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/metainfo.xml new file mode 100644 index 0000000000..3167565f2e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/metainfo.xml @@ -0,0 +1,99 @@ +<?xml version="1.0"?> +<!--Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>DINKY</name> + <displayName>Dinky</displayName> + <comment>Dinky is a distributed multi-tenant Thrift JDBC/ODBC server for large-scale data management, processing, and analytics, built on top of Apache Spark and designed to support more engines (i.e., Flink). </comment> + <version>1.0.0</version> + <components> + <component> + <name>DINKY_SERVER</name> + <displayName>Dinky Server</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/dinky_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <configFiles> + <configFile> + <type>env</type> + <fileName>dinky-application-server.xml</fileName> + <dictionaryName>dinky-defaults</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>dinky-env.xml</fileName> + <dictionaryName>dinky-env</dictionaryName> + </configFile> + </configFiles> + <logs> + <log> + <logId>dinky_server</logId> + <primary>true</primary> + </log> + </logs> + </component> + </components> + + <configuration-dependencies> + <config-type>dinky-application-server</config-type> + <config-type>dinky-env</config-type> + </configuration-dependencies> + + <quickLinksConfigurations> + <quickLinksConfiguration> + <fileName>quicklinks.json</fileName> + <default>true</default> + </quickLinksConfiguration> + </quickLinksConfigurations> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <osSpecifics> + <osSpecific> + <osFamily>redhat9,redhat8,redhat7,amazonlinux2,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>dinky_${stack_version}</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,debian9,ubuntu12,ubuntu14,ubuntu16,ubuntu18</osFamily> + <packages> + <package> + <name>dinky_${stack_version}</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + </service> + </services> +</metainfo> diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/dinky_env.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/dinky_env.py new file mode 100644 index 0000000000..6e1ece7055 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/dinky_env.py @@ -0,0 +1,70 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import os + +def dinky_env(): + import params # import 导入params.py文件 + # User(params.dinky_user, action="create", groups=params.dinky_group + # , ignore_already_exists=True # 忽略已经存在 + # ) # 创建用户 + + Directory([params.dinky_log_dir, params.dinky_pid_dir], + owner=params.dinky_user, + group=params.dinky_group, + mode=0o775, + create_parents = True + ) + + + ## 重写 启动脚本 auto.sh 文件 + File(os.path.join(params.dinky_bin_dir, params.start_script_name), + mode=0o755, + content=Template(params.start_script_template_file), + owner=params.dinky_user, + group=params.dinky_group + ) + + #content=Template("hbase.conf.j2") + # 重写 application.yml 文件 + File(os.path.join(params.dinky_conf_dir, params.dinky_application_main_config_file), + mode=0o755, + content=Template(params.dinky_application_main_config_template_file), + owner=params.dinky_user, + group=params.dinky_group + ) + + ## 根据 mysql 还是 pgsql 重写 application-xxx.yml 文件 + if params.dinky_database_config['dinky_database_type'] == "mysql": + ## 重写 application-mysql.yml 文件 + File(os.path.join(params.dinky_conf_dir , params.dinky_application_mysql_config_file), + mode=0o755, + content=Template(params.dinky_application_mysql_config_template_file), + owner=params.dinky_user, + group=params.dinky_group + ) + else: + ## 重写 application-pgsql.yml 文件 + File(os.path.join(params.dinky_conf_dir , params.dinky_application_pgsql_config_file), + mode=0o755, + content=Template(params.dinky_application_pgsql_config_template_file), + owner=params.dinky_user, + group=params.dinky_group + ) \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/dinky_server.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/dinky_server.py new file mode 100644 index 0000000000..63a8c93d2b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/dinky_server.py @@ -0,0 +1,89 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import time +import os +from resource_management import * +from resource_management.libraries.functions.format import format +from dinky_env import dinky_env + + +class DinkyService(Script): + + # 安装 dinky + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + #Execute(('chmod', '-R', '777', params.dinky_home)) + #Execute(('chown', '-R', params.dinky_user + ":" + params.dinky_group, params.dinky_home)) + + + def initialize(self, env): + import params + env.set_params(params) + dinky_setup_marker = os.path.join(params.dinky_conf_dir, "dinky_setup") + if not os.path.exists(dinky_setup_marker): + try : + Execute(params.init_sql, user=params.dinky_user) + Logger.info(format('dinky init finished, cmd: {params.init_sql}')) + + File(dinky_setup_marker, + owner = params.dinky_user, + group = params.dinky_group, + mode = 0o640) + except Exception as e: + Logger.exception("There was an exception when ALTER SYSTEM ADD FOLLOWER: " + str(e)) + + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + self.initialize(env) + dinky_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + no_op_test = format("ls {params.dinky_pid_file} >/dev/null 2>&1 ") + + start_cmd = format("sh {params.start_script_path} start {params.dinky_flink_big_version}") + Execute(start_cmd, user=params.dinky_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh {params.start_script_path} stop ") + Execute(stop_cmd, user=params.dinky_user) + time.sleep(5) + + def status(self, env): + import params + env.set_params(params) + check_process_status(params.dinky_pid_file) + + def restart(self, env): + self.stop(env) + self.start(env) + + +if __name__ == "__main__": + DinkyService().execute() diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/params.py new file mode 100644 index 0000000000..a4fa11c2cd --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/params.py @@ -0,0 +1,108 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import sys +import os +from resource_management import * +from resource_management.core.logger import Logger +from resource_management.libraries.functions.format import format +from resource_management.libraries.script.script import Script + + + +config = Script.get_config() + +stack_root = Script.get_stack_root() + +component_directory = "dinky-server" +dinky_home = format("{stack_root}/current/{component_directory}") +dinky_conf_dir = format("{dinky_home}/config") + + +dinky_user = config['configurations']['dinky-env']['dinky_user'] +dinky_group = config['configurations']['dinky-env']['dinky_group'] +dinky_pid_dir = config['configurations']['dinky-env']['dinky_pid_dir'] +dinky_pid_filename = format("dinky-{dinky_user}.pid") +dinky_pid_file = os.path.join(dinky_pid_dir, dinky_pid_filename) + +dinky_log_dir = config['configurations']['dinky-env']['dinky_log_dir'] + + + +dinky_bin_dir = dinky_home +dinky_lib_dir = os.path.join(dinky_home, "lib") +dinky_lib_jars = format("{dinky_lib_dir}/*") +dinky_extends_flink_dir = os.path.join(dinky_home, "extends") +dinky_extends_flink_jars = format("{dinky_extends_flink_dir}/*") + +dinky_init_mysql_sqlfile = os.path.join(dinky_home, "sql/dinky-mysql.sql") +dinky_init_pgsqll_sqlfile = os.path.join(dinky_home, "sql/dinky-pg.sql") +dinky_application_main_config_file = "application.yml" +dinky_application_main_config_template_file = format("{dinky_application_main_config_file}.j2") +dinky_application_mysql_config_file = "application-mysql.yml" +dinky_application_mysql_config_template_file = format("{dinky_application_mysql_config_file}.j2") +dinky_application_pgsql_config_file = "application-pgsql.yml" +dinky_application_pgsql_config_template_file = format("{dinky_application_pgsql_config_file}.j2") + +start_script_name = "auto.sh" +start_script_path = os.path.join(dinky_home, start_script_name) +start_script_template_file = format("{start_script_name}.j2") + + + +dinky_env_map = {} + +dinky_env_map.update(config['configurations']['dinky-application-server']) +dinky_flink_big_version = dinky_env_map['flink.big.version'] + +dinky_server_port = dinky_env_map['server.port'] + +dinky_database_config = {'dinky_database_type': dinky_env_map['spring.profiles.active'], + 'dinky_database_username': dinky_env_map['spring.datasource.database.username'], + 'dinky_database_password': dinky_env_map['spring.datasource.database.password']} + + +dinky_database_password = dinky_env_map['spring.datasource.database.password'] +dinky_database_name = dinky_env_map['spring.datasource.database.name'] +if 'mysql' == dinky_database_config['dinky_database_type']: + + dinky_database_config['dinky_database_driver'] = 'com.mysql.jdbc.Driver' + dinky_database_config['dinky_database_url'] = 'jdbc:mysql://' + dinky_env_map['spring.datasource.database.host'] \ + + ':' + dinky_env_map['spring.datasource.database.port'] \ + + '/' + dinky_env_map['spring.datasource.database.name'] \ + + '?useUnicode=true&characterEncoding=UTF-8' + database_host = dinky_env_map['spring.datasource.database.host'] + database_port = dinky_env_map['spring.datasource.database.port'] + dinky_init_sql_path = dinky_init_mysql_sqlfile + sql_client = "mysql" + #mysql -h hostname -P port -u username -p'password' dinky_database_name < /path/to/file.sql + + init_sql = format("{sql_client} -h {database_host} -P {database_port} -u {dinky_user} -p'{dinky_database_password}' {dinky_database_name} < {dinky_init_sql_path}") +else: + + dinky_database_config['dinky_database_driver'] = 'org.postgresql.Driver' + dinky_database_config['dinky_database_url'] = 'jdbc:postgresql://' + dinky_env_map[ + 'spring.datasource.database.host'] \ + + ':' + dinky_env_map['spring.datasource.database.port'] \ + + '/' + dinky_env_map['spring.datasource.database.name'] \ + + '?stringtype=unspecified' + database_host = dinky_env_map['spring.datasource.database.host'] + database_port = dinky_env_map['spring.datasource.database.port'] + dinky_init_sql_path = dinky_init_pgsqll_sqlfile + sql_client = "psql" + init_sql = format("PGPASSWORD={dinky_database_password} {sql_client} -h {database_host} -p {database_port} -U {dinky_user} -d {dinky_database_name} < {dinky_init_sql_path}") diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/service_check.py new file mode 100644 index 0000000000..9c9d23bff0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/scripts/service_check.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + + +from resource_management import * + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + Execute(format("sh {dinky_bin_dir}/{start_script_name} status")) + +if __name__ == "__main__": + ServiceCheck().execute() \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application-mysql.yml.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application-mysql.yml.j2 new file mode 100644 index 0000000000..73864dfc80 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application-mysql.yml.j2 @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +spring: + datasource: + driver-class-name: {{ dinky_database_config['dinky_database_driver'] }} + url: {{ dinky_database_config['dinky_database_url'] }} + username: {{ dinky_database_config['dinky_database_username'] }} + password: {{ dinky_database_config['dinky_database_password'] }} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application-pgsql.yml.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application-pgsql.yml.j2 new file mode 100644 index 0000000000..3a158c08ec --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application-pgsql.yml.j2 @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + datasource: + driver-class-name: {{ dinky_database_config['dinky_database_driver'] }} + url: {{ dinky_database_config['dinky_database_url'] }} + username: {{ dinky_database_config['dinky_database_username'] }} + password: {{ dinky_database_config['dinky_database_password'] }} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application.yml.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application.yml.j2 new file mode 100644 index 0000000000..2efa79ea48 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/application.yml.j2 @@ -0,0 +1,153 @@ +################################################################################################################# +################################################# Common Config ################################################# +################################################################################################################# +# Dinky application port +server: + port: {{ dinky_server_port }} + +spring: + # Dinky application name + application: + name: Dinky + profiles: + # The h2 database is used by default. If you need to use other databases, please set the configuration active to: mysql, currently supports [mysql, pgsql, h2] + # If you use mysql database, please configure mysql database connection information in application-mysql.yml + # If you use pgsql database, please configure pgsql database connection information in application-pgsql.yml + # If you use the h2 database, please configure the h2 database connection information in application-h2.yml, + # note: the h2 database is only for experience use, and the related data that has been created cannot be migrated, please use it with caution + active: {{ dinky_database_config['dinky_database_type'] }} #[mysql,pgsql] + include: jmx + + # mvc config + mvc: + pathmatch: + # Path matching strategy, default ant_path_matcher, support ant_path_matcher and path_pattern_parser + matching-strategy: ant_path_matcher + format: + date: yyyy-MM-dd HH:mm:ss # date format + time: HH:mm:ss # time format + date-time: yyyy-MM-dd HH:mm:ss # date-time format + + # json format global configuration + jackson: + time-zone: GMT+8 # Time zone, default is GMT+8 + date-format: yyyy-MM-dd HH:mm:ss # Date format, the default is yyyy-MM-dd HH:mm:ss + + # circular references allowed + main: + allow-circular-references: true + + # file upload config of servlet , the default is 500MB + servlet: + multipart: + enabled: true + max-file-size: 524288000 + max-request-size: 524288000 + + + # By default, memory cache metadata information is used, + # dinky supports redis cache, if necessary, please change simple to redis, and open the following redis connection configuration + # Sub-configuration items can be opened or customized as needed + cache: + type: simple + # If type is configured as redis, this item can be configured as needed, note: Pay attention to the indentation of this configuration item + # redis: + # # Whether to cache empty values, save the default + # cache-null-values: false + # # Cache expiration time, default 24 hours + # time-to-live: 86400 + + ########################################################## Redis配置 ########################################################## + # If sa-token needs to rely on redis, please open the redis configuration and depend on pom.xml and dinky-admin/pom.xml, and configure redis connection information in application.yml + # note: pay attention to the indentation of this configuration item +# redis: +# host: localhost +# port: 6379 +# password: +# database: 10 +# jedis: +# pool: +# # The maximum number of connections in the connection pool (use a negative value to indicate no limit) +# max-active: 50 +# # The maximum blocking waiting time of the connection pool (use a negative value to indicate no limit) +# max-wait: 3000 +# # The maximum number of idle connections in the connection pool +# max-idle: 20 +# # The minimum number of idle connections in the connection pool +# min-idle: 5 +# # Connection timeout (milliseconds) +# timeout: 5000 + +################################################################################################################# +################################################# Mybatis Config ################################################ +######### Please note: that the following configurations are not recommended to be modified ##################### +################################################################################################################# +mybatis-plus: + mapper-locations: classpath:/mapper/*Mapper.xml + # Entity scanning, multiple packages are separated by commas or semicolons + typeAliasesPackage: org.dinky.model + global-config: + db-config: + id-type: auto + # Logic delete configuration : 0: false(Not deleted), 1: true(deleted) + logic-delete-field: is_delete + logic-delete-value: 1 + logic-not-delete-value: 0 + banner: false + configuration: + ##### mybatis-plus prints complete sql (only for development environment) + #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + log-impl: org.apache.ibatis.logging.nologging.NoLoggingImpl + type-handlers-package: org.dinky.data.typehandler +################################################################################################################# +################################################# SMS Config #################################################### +################################################################################################################# +sms: + is-print: false + + + +################################################################################################################# +################################################# Sa-Token Config ############################################### +################################################################################################################# +# Sa-Token basic configuration +sa-token: + # The validity period of the token, the unit is 10 hours by default, -1 means it will never expire + timeout: 36000 + # The temporary validity period of the token (the token will be considered as expired if there is no operation within the specified time) + # unit: second , if you do not need to set a temporary token, you can set it to -1 + active-timeout: 1800 + # Whether to allow the same account to log in concurrently (when true, allow login together, when false, new login squeezes out old login) + is-concurrent: false + # When multiple people log in to the same account, whether to share a token (if true, all logins share a token, and if false, create a new token for each login) + is-share: true + # token style + token-style: uuid + # Whether to output the operation log + is-log: false + # Whether to print banner + is-print: false + # The secret key + jwt-secret-key: 0DA4198858E84F1AADDF846340587A85 + # is write header + is-write-header: true + # is read header + is-read-header: true + token-name: token + +################################################################################################################# +################################################# knife4j Config ################################################ +################################################################################################################# +knife4j: + enable: true + setting: + language: en + + + +################################################################################################################# +################################################# Crypto Config ################################################# +################################################################################################################# +crypto: + enabled: false + encryption-password: diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/auto.sh.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/auto.sh.j2 new file mode 100644 index 0000000000..4dd401c36d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/package/templates/auto.sh.j2 @@ -0,0 +1,148 @@ +#!/bin/bash + +FLINK_VERSION={{ dinky_flink_big_version }} + +JAR_NAME="dinky-admin" + +PID_FILE={{ dinky_pid_filename }} + +# JMX path +APP_HOME="$(cd `dirname $0`; pwd)" + +# Use FLINK_HOME: +CLASS_PATH="${APP_HOME}:${APP_HOME}/lib/*:${APP_HOME}/config:${APP_HOME}/extends/*:${APP_HOME}/plugins/*:${APP_HOME}/customJar/*:${APP_HOME}/plugins/flink${FLINK_VERSION}/dinky/*:${APP_HOME}/plugins/flink${FLINK_VERSION}/*:${APP_HOME}/extends/flink${FLINK_VERSION}/dinky/*:${APP_HOME}/extends/flink${FLINK_VERSION}/*" + + +JMX="-javaagent:$APP_HOME/lib/jmx_prometheus_javaagent-0.20.0.jar=10087:$APP_HOME/config/jmx/jmx_exporter_config.yaml" + +# Check whether the pid path exists +PID_PATH={{ dinky_pid_dir }} + +if [ -d "${PID_PATH}" ];then + echo "${PID_PATH} is already exist." >> /dev/null +else + mkdir -p "${PID_PATH}" +fi + +# Check whether the pid file exists +if [ -f "${PID_PATH}/${PID_FILE}" ];then + echo "${PID_PATH}/${PID_FILE} is already exist." >> /dev/null +else + touch "${PID_PATH}"/${PID_FILE} +fi + +tips() { + echo "" + echo "WARNING!!!......Tips, please use command: sh auto.sh [start|startOnPending|startWithJmx|stop|restart|restartWithJmx|status]. For example: sh auto.sh start " + echo "" + exit 1 +} + +updatePid() { + pid=$(ps -ef | grep [D]inky | awk '{print $2}' | head -1) + echo $pid >"${PID_PATH}"/${PID_FILE} +} + +start() { + updatePid + if [ -z "$pid" ]; then + nohup java -Ddruid.mysql.usePingMethod=false -Dlog4j2.isThreadContextMapInheritable=true -Xms512M -Xmx2048M -XX:PermSize=512M -XX:MaxPermSize=1024M -XX:+HeapDumpOnOutOfMemoryError -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky & + echo $! >"${PID_PATH}"/${PID_FILE} + echo "FLINK VERSION : $FLINK_VERSION" + echo "........................................Start Dinky Successfully........................................" + else + echo "Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!" + fi +} + +startOnPending() { + updatePid + if [ -z "$pid" ]; then + java -Ddruid.mysql.usePingMethod=false -Xms512M -Xmx2048M -XX:PermSize=512M -XX:MaxPermSize=1024M -XX:+HeapDumpOnOutOfMemoryError -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky + echo "FLINK VERSION : $FLINK_VERSION" + echo "........................................Start Dinky Successfully........................................" + else + echo "Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!" + fi +} + +startWithJmx() { + updatePid + if [ -z "$pid" ]; then + nohup java -Ddruid.mysql.usePingMethod=false -Xms512M -Xmx2048M -XX:PermSize=512M -XX:MaxPermSize=1024M -XX:+HeapDumpOnOutOfMemoryError -Xverify:none "${JMX}" -cp "${CLASS_PATH}" org.dinky.Dinky & +# echo $! >"${PID_PATH}"/${PID_FILE} + updatePid + echo "........................................Start Dinky with Jmx Successfully..................................... + ..." + else + echo "Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!" + fi +} + +stop() { + updatePid + pid=$(cat "${PID_PATH}"/${PID_FILE}) + if [ -z $pid ]; then + echo "Dinky pid is not exist in ${PID_PATH}/${PID_FILE}" + else + kill -9 $pid + sleep 1 + echo "........................................Stop Dinky Successfully....................................." + rm -f "${PID_PATH}"/${PID_FILE} + fi +} + +status() { + updatePid + if [ -z $pid ]; then + echo "" + echo "Service ${JAR_NAME} is not running!" + echo "" + exit 1 + else + echo "" + echo "Service ${JAR_NAME} is running. It's pid=${pid}" + echo "" + fi +} + +restart() { + echo "" + stop + start + echo "........................................Restart Successfully........................................" +} + +restartWithJmx() { + echo "" + stop + startWithJmx + echo "........................................Restart with Jmx Successfully........................................" +} + +case "$1" in +"start") + start + ;; +"startOnPending") + startOnPending + ;; +"startWithJmx") + startWithJmx + ;; +"stop") + stop + ;; +"status") + status + ;; +"restart") + restart + ;; +"restartWithJmx") + restartWithJmx + ;; +*) + tips + ;; +esac diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/quicklinks/quicklinks.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/quicklinks/quicklinks.json new file mode 100644 index 0000000000..ec5e39e9fb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/quicklinks/quicklinks.json @@ -0,0 +1,26 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"http" + }, + + "links": [ + { + "name": "dinky-application-ui", + "label": "Dinky Web UI", + "requires_user_name": "false", + "component_name": "DINKY_SERVER", + "url": "%@://%@:%@", + "port":{ + "http_property": "server.port", + "http_default_port": "8888", + "regex": "^(\\d+)$", + "site": "dinky-application-ui" + } + } + ] + } +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/service_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/service_advisor.py new file mode 100755 index 0000000000..c3f5586777 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/DINKY/service_advisor.py @@ -0,0 +1,196 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +from ast import Param +import imp +import os +import traceback +import re +import socket +import fnmatch + + +from resource_management.core.logger import Logger + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + if "BASE_SERVICE_ADVISOR" in os.environ: + PARENT_FILE = os.environ["BASE_SERVICE_ADVISOR"] + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +class DinkyServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(DinkyServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + + # Nothing to do + pass + + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return self.getServiceComponentCardinalityValidations(services, hosts, "KYUUBI") + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + """ + Entry point. + Must be overriden in child class. + """ + # Logger.info("Class: %s, Method: %s. Recommending Service Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + recommender = DinkyRecommender() + recommender.recommendDinkyConfigurationsFromHDP33(configurations, clusterData, services, hosts) + + + # def getServiceConfigurationRecommendationsForSSO(self, configurations, clusterData, services, hosts): + # """ + # Entry point. + # Must be overriden in child class. + # """ + # recommender = DinkyRecommender() + # recommender.recommendConfigurationsForSSO(configurations, clusterData, services, hosts) + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + # Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + return [] + + @staticmethod + def isKerberosEnabled(services, configurations): + """ + Determines if security is enabled by testing the value of core-site/hadoop.security.authentication enabled. + If the property exists and is equal to "kerberos", then is it enabled; otherwise is it assumed to be + disabled. + + :type services: dict + :param services: the dictionary containing the existing configuration values + :type configurations: dict + :param configurations: the dictionary containing the updated configuration values + :rtype: bool + :return: True or False + """ + if configurations and "core-site" in configurations and \ + "hadoop.security.authentication" in configurations["core-site"]["properties"]: + return configurations["core-site"]["properties"]["hadoop.security.authentication"].lower() == "kerberos" + elif services and "core-site" in services["configurations"] and \ + "hadoop.security.authentication" in services["configurations"]["core-site"]["properties"]: + return services["configurations"]["core-site"]["properties"]["hadoop.security.authentication"].lower() == "kerberos" + else: + return False + + +class DinkyRecommender(service_advisor.ServiceAdvisor): + """ + Dinky Recommender suggests properties when adding the service for the first time or modifying configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(DinkyRecommender, self) + self.as_super.__init__(*args, **kwargs) + + def recommendDinkyConfigurationsFromHDP33(self, configurations, clusterData, services, hosts): + """ + Recommend configurations for this service based on HDP 3.3. + """ + + + + + + + + + + + --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
