AMBARI-17981. Integrate Druid with Ambari (Nishant Bangarwa, Slim Bouguerra via Swapan Shridhar).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dcbd2c07 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dcbd2c07 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dcbd2c07 Branch: refs/heads/trunk Commit: dcbd2c07c8fdb9db22d11ac94aea3df9bc27595c Parents: d31c617 Author: Swapan Shridhar <[email protected]> Authored: Mon Oct 24 13:47:51 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Mon Oct 24 13:47:51 2016 -0700 ---------------------------------------------------------------------- .../DRUID/0.9.2/configuration/druid-broker.xml | 100 +++ .../DRUID/0.9.2/configuration/druid-common.xml | 206 ++++++ .../0.9.2/configuration/druid-coordinator.xml | 43 ++ .../DRUID/0.9.2/configuration/druid-env.xml | 241 +++++++ .../0.9.2/configuration/druid-historical.xml | 88 +++ .../DRUID/0.9.2/configuration/druid-log4j.xml | 84 +++ .../0.9.2/configuration/druid-logrotate.xml | 46 ++ .../0.9.2/configuration/druid-middlemanager.xml | 104 +++ .../0.9.2/configuration/druid-overlord.xml | 52 ++ .../DRUID/0.9.2/configuration/druid-router.xml | 59 ++ .../common-services/DRUID/0.9.2/metainfo.xml | 249 +++++++ .../DRUID/0.9.2/package/scripts/broker.py | 28 + .../DRUID/0.9.2/package/scripts/coordinator.py | 28 + .../DRUID/0.9.2/package/scripts/druid.py | 259 ++++++++ .../DRUID/0.9.2/package/scripts/druid_node.py | 85 +++ .../DRUID/0.9.2/package/scripts/historical.py | 28 + .../0.9.2/package/scripts/middlemanager.py | 28 + .../DRUID/0.9.2/package/scripts/overlord.py | 28 + .../DRUID/0.9.2/package/scripts/params.py | 129 ++++ .../DRUID/0.9.2/package/scripts/router.py | 28 + .../0.9.2/package/scripts/service_check.py | 44 ++ .../0.9.2/package/scripts/status_params.py | 24 + .../DRUID/0.9.2/quicklinks/quicklinks.json | 37 ++ .../DRUID/0.9.2/themes/theme.json | 120 ++++ .../stacks/HDP/2.6/role_command_order.json | 19 + .../stacks/HDP/2.6/services/DRUID/kerberos.json | 78 +++ .../stacks/HDP/2.6/services/DRUID/metainfo.xml | 27 + .../stacks/HDP/2.6/services/stack_advisor.py | 187 +++++- .../test/python/stacks/2.6/DRUID/test_druid.py | 647 +++++++++++++++++++ .../stacks/2.6/common/test_stack_advisor.py | 575 ++++++++++++++++ .../test/python/stacks/2.6/configs/default.json | 552 ++++++++++++++++ 31 files changed, 4222 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml new file mode 100644 index 0000000..4f05da0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml @@ -0,0 +1,100 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>druid.service</name> + <value>druid/broker</value> + <description>The druid.service name of broker node.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.port</name> + <value>8082</value> + <description>The port on which the broker will accept connections.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.broker.http.numConnections</name> + <value>20</value> + <description>Size of connection pool for the Broker to connect to historical and real-time nodes. If there are more + queries than this number that all need to speak to the same node, then they will queue up. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.server.http.numThreads</name> + <value>50</value> + <description>Number of threads for HTTP requests.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.processing.buffer.sizeBytes</name> + <value>1073741824</value> + <value-attributes> + <type>long</type> + <minimum>0</minimum> + <unit>Bytes</unit> + </value-attributes> + <description>This specifies a buffer size for the storage of intermediate results. The computation engine in both + the Historical and Realtime nodes will use a scratch buffer of this size to do all of their intermediate + computations + off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can + require more passes depending on the query that is being executed. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.processing.numThreads</name> + <value>2</value> + <description>The number of processing threads to have available for parallel processing of segments.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.broker.cache.useCache</name> + <value>true</value> + <description>Enable the cache on the broker.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.broker.cache.populateCache</name> + <value>true</value> + <description>Populate the cache on the broker.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.cache.type</name> + <value>local</value> + <description>The type of cache to use for queries.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.cache.sizeInBytes</name> + <value>10000000</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <unit>Bytes</unit> + </value-attributes> + <description>Maximum cache size in bytes. Zero disables caching.</description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml new file mode 100644 index 0000000..8712c7d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml @@ -0,0 +1,206 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>druid.extensions.pullList</name> + <value>[]</value> + <description>A comma-separated list of one or more druid extensions to download from maven.</description> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.type</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.extensions.loadList</name> + <value>["druid-datasketches"] + </value> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.type</name> + </property> + </depends-on> + <description>A comma-separated list of one or more druid extensions to load.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.zk.service.host</name> + <value>localhost:2181</value> + <description> + zookeeper connection string. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.zk.paths.base</name> + <value>/druid</value> + <description> + Base Zookeeper path + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.discovery.curator.path</name> + <value>/druid/discovery</value> + <description> + Services announce themselves under this ZooKeeper path. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.storage.type</name> + <value></value> + <description> + Choices:local, noop, s3, hdfs, c*. The type of deep storage to use. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.storage.storageDirectory</name> + <value></value> + <description> + directory to use as deep storage. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.metadata.storage.connector.password</name> + <value></value> + <property-type>PASSWORD</property-type> + <display-name>Metadata storage password</display-name> + <description>Password for the metadata storage data base.</description> + <value-attributes> + <type>password</type> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.metadata.storage.connector.user</name> + <value>druid</value> + <display-name>Metadata storage user</display-name> + <description>Metadata storage user</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.metadata.storage.connector.port</name> + <value>1527</value> + <display-name>Metadata storage port</display-name> + <description>Metadata storage port</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.type</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>database_name</name> + <value>druid</value> + <display-name>Metadata storage database name</display-name> + <description>Metadata storage database name</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>metastore_hostname</name> + <value>localhost</value> + <display-name>Metadata storage hostname name</display-name> + <description>Metadata storage hostname name</description> + <on-ambari-upgrade add="false"/> + </property> + <property require-input="true"> + <name>druid.metadata.storage.type</name> + <display-name>Metadata storage type</display-name> + <value>derby</value> + <value-attributes> + <overridable>false</overridable> + <type>value-list</type> + <entries> + <entry> + <value>mysql</value> + <label>MYSQL</label> + </entry> + <entry> + <value>derby</value> + <label>DERBY</label> + </entry> + <entry> + <value>postgres</value> + <label>POSTGRES</label> + </entry> + </entries> + </value-attributes> + <description>Type of the metadata storage. Note that derby will work only if all the druid node are located + within the same node. Use mysql or postgres for distributed mode. + mysql installed by ambari is only for development and not suitable for production use cases due to it being not HA + </description> + <on-ambari-upgrade add="false"/> + </property> + <property require-input="true"> + <name>druid.metadata.storage.connector.connectURI</name> + <value>jdbc:derby://localhost:1527/druid;create=true</value> + <display-name>Metadata storage connector url</display-name> + <description>Metadata storage connector url</description> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>druid-common</type> + <name>database_name</name> + </property> + <property> + <type>druid-common</type> + <name>metastore_hostname</name> + </property> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.type</name> + </property> + <property> + <type>druid-common</type> + <name>druid.metadata.storage.connector.port</name> + </property> + </depends-on> + </property> + <property> + <name>druid.hadoop.security.kerberos.principal</name> + <display-name>kerberos principal</display-name> + <description>Kerberos principal e.g [email protected]</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.hadoop.security.kerberos.keytab</name> + <display-name>Kerberos keytab location</display-name> + <description>Kerberos keytab location</description> + <on-ambari-upgrade add="false"/> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-coordinator.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-coordinator.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-coordinator.xml new file mode 100644 index 0000000..618f11d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-coordinator.xml @@ -0,0 +1,43 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>druid.service</name> + <value>druid/coordinator</value> + <description>The druid.service name of coordinator node.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.port</name> + <value>8081</value> + <description>The port on which the coordinator will accept connections.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.coordinator.merge.on</name> + <value>false</value> + <description>Boolean flag for whether or not the coordinator should try and merge small segments into a more optimal + segment size. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-env.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-env.xml new file mode 100644 index 0000000..fa24cac --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-env.xml @@ -0,0 +1,241 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <!--Heap Settings --> + <property> + <name>druid.broker.jvm.heap.memory</name> + <value>2048</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>65536</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.coordinator.jvm.heap.memory</name> + <value>512</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>65536</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.middlemanager.jvm.heap.memory</name> + <value>256</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>65536</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.historical.jvm.heap.memory</name> + <value>2048</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>65536</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.overlord.jvm.heap.memory</name> + <value>512</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>65536</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.router.jvm.heap.memory</name> + <value>512</value> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>65536</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <!-- DirectMemorySettings --> + <property> + <name>druid.broker.jvm.direct.memory</name> + <value>1048576</value> + <depends-on> + <property> + <type>druid-broker</type> + <name>druid.processing.buffer.sizeBytes</name> + </property> + <property> + <type>druid-broker</type> + <name>druid.processing.numThreads</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.coordinator.jvm.direct.memory</name> + <value>1048576</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.middlemanager.jvm.direct.memory</name> + <value>1048576</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.historical.jvm.direct.memory</name> + <value>1048576</value> + <depends-on> + <property> + <type>druid-historical</type> + <name>druid.processing.buffer.sizeBytes</name> + </property> + <property> + <type>druid-historical</type> + <name>druid.processing.numThreads</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.overlord.jvm.direct.memory</name> + <value>1048576</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.router.jvm.direct.memory</name> + <value>1048576</value> + <on-ambari-upgrade add="false"/> + </property> + + <!-- JavaOpts Tune GC related configs here--> + <property> + <name>druid.broker.jvm.opts</name> + <value>-Duser.timezone=UTC -Dfile.encoding=UTF-8</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.coordinator.jvm.opts</name> + <value>-Duser.timezone=UTC -Dfile.encoding=UTF-8</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.middlemanager.jvm.opts</name> + <value>-Duser.timezone=UTC -Dfile.encoding=UTF-8</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.historical.jvm.opts</name> + <value>-Duser.timezone=UTC -Dfile.encoding=UTF-8</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.overlord.jvm.opts</name> + <value>-Duser.timezone=UTC -Dfile.encoding=UTF-8</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.router.jvm.opts</name> + <value>-Duser.timezone=UTC -Dfile.encoding=UTF-8</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid_user</name> + <display-name>Druid User</display-name> + <value>druid</value> + <property-type>USER</property-type> + <description></description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid_log_dir</name> + <value>/var/log/druid</value> + <description></description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid_pid_dir</name> + <value>/var/run/druid</value> + <display-name>Druid PID dir</display-name> + <description></description> + <value-attributes> + <type>directory</type> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <!-- druid-env.sh --> + <property> + <name>content</name> + <display-name>druid-env template</display-name> + <description>This is simple template for druid-env.sh file</description> + <value> + #!/bin/bash + + # Set DRUID specific environment variables here. + + # The java implementation to use. + export JAVA_HOME={{java8_home}} + export PATH=$PATH:$JAVA_HOME/bin + export DRUID_PID_DIR={{druid_pid_dir}} + export DRUID_LOG_DIR={{druid_log_dir}} + export DRUID_CONF_DIR={{druid_conf_dir}} + export DRUID_LIB_DIR={{druid_home}}/lib + + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml new file mode 100644 index 0000000..0545e4c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml @@ -0,0 +1,88 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>druid.service</name> + <value>druid/historical</value> + <description>The druid.service name of historical node.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.port</name> + <value>8083</value> + <description>The port on which the historical nodes will accept connections.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.server.http.numThreads</name> + <value>50</value> + <description>Number of threads for HTTP requests.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.processing.buffer.sizeBytes</name> + <value>1073741824</value> + <value-attributes> + <type>long</type> + <minimum>0</minimum> + <unit>Bytes</unit> + </value-attributes> + <description>This specifies a buffer size for the storage of intermediate results. The computation engine in both + the Historical and Realtime nodes will use a scratch buffer of this size to do all of their intermediate + computations off-heap. Larger values allow for more aggregations in a single pass over the data while smaller + values can require more passes depending on the query that is being executed. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.processing.numThreads</name> + <value>10</value> + <description>The number of processing threads to have available for parallel processing of segments.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.segmentCache.locations</name> + <value>[{"path":"/apps/druid/segmentCache","maxSize"\:300000000000}]</value> + <description>Segments assigned to a Historical node are first stored on the local file system (in a disk cache) and + then served by the Historical node. These locations define where that local cache resides. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.server.maxSize</name> + <value>300000000000</value> + <description>The maximum number of bytes-worth of segments that the node wants assigned to it. This is not a limit + that Historical nodes actually enforces, just a value published to the Coordinator node so it can plan + accordingly. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.segmentCache.infoDir</name> + <value>/apps/druid/segmentCache</value> + <description>Historical nodes keep track of the segments they are serving so that when the process is restarted they + can reload the same segments without waiting for the Coordinator to reassign. This path defines where this + metadata is kept. Directory will be created if needed. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-log4j.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-log4j.xml new file mode 100644 index 0000000..bcb731a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-log4j.xml @@ -0,0 +1,84 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>druid_log_level</name> + <value>info</value> + <description>Log level for io.druid logging</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>root_log_level</name> + <value>WARN</value> + <description>Log level for root logging</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>metamx_log_level</name> + <value>info</value> + <description>Log level for com.metamxlogging</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>content</name> + <display-name>druid-log4j template</display-name> + <description>Custom log4j.properties</description> + <value><![CDATA[<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + <Configuration> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/> + </Console> + </Appenders> + <Loggers> + <Logger name="com.metamx" level="{{metamx_log_level}}"/> + <Logger name="io.druid" level="{{druid_log_level}}"/> + <Root level="{{root_log_level}}"> + <AppenderRef ref="Console"/> + </Root> + </Loggers> + </Configuration> + ]]></value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml new file mode 100644 index 0000000..f612766 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml @@ -0,0 +1,46 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>druid logrotate template</display-name> + <description>Custom logrotate file</description> + <value><![CDATA[ + {{druid_log_dir}}/*.log { + copytruncate + rotate 7 + daily + nocompress + missingok + notifempty + create 660 druid users + dateext + dateformat -%Y-%m-%d-%s + } + ]]></value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml new file mode 100644 index 0000000..967438e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml @@ -0,0 +1,104 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>druid.service</name> + <value>druid/middlemanager</value> + <description>The druid.service name of middlemanager node.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.port</name> + <value>8091</value> + <description>The port on which the middlemanager nodes will accept connections.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.indexer.runner.startPort</name> + <value>8100</value> + <description>The port that peons begin running on.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.worker.capacity</name> + <value>3</value> + <description> + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.indexer.runner.javaOpts</name> + <value>-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dhdp.version={{stack_version}} -Dhadoop.mapreduce.job.classloader=true</value> + <description> + A string of -X Java options to pass to the peon's JVM. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.indexer.task.baseTaskDir</name> + <value>/tmp/persistent/tasks</value> + <description> + Base temporary working directory for druid tasks. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.server.http.numThreads</name> + <value>50</value> + <description> + Number of threads for HTTP requests. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.processing.buffer.sizeBytes</name> + <value>256000000</value> + <value-attributes> + <type>long</type> + <minimum>0</minimum> + <unit>Bytes</unit> + </value-attributes> + <description> + This specifies a buffer size for the storage of intermediate results. The computation engine in both the + Historical and Realtime nodes will use a scratch buffer of this size to do all of their intermediate computations + off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can + require more passes depending on the query that is being executed. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.processing.numThreads</name> + <value>2</value> + <description> + The number of processing threads to have available for parallel processing of segments. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.indexer.task.hadoopWorkingPath</name> + <value>/tmp/druid-indexing</value> + <description> + Temporary working directory for Hadoop tasks + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-overlord.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-overlord.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-overlord.xml new file mode 100644 index 0000000..57d1c63 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-overlord.xml @@ -0,0 +1,52 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>druid.service</name> + <value>druid/overlord</value> + <description>The druid.service name of overlord node.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.port</name> + <value>8090</value> + <description>The port on which the overlord will accept connections.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.indexer.runner.type</name> + <value>remote</value> + <description>Choices "local" or "remote". Indicates whether tasks should be run locally or in a distributed + environment. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.indexer.storage.type</name> + <value>metadata</value> + <description>Choices are "local" or "metadata". Indicates whether incoming tasks should be stored locally (in heap) + or in metadata storage. Storing incoming tasks in metadata storage allows for tasks to be resumed if the overlord + should fail. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-router.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-router.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-router.xml new file mode 100644 index 0000000..bfb0d21 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-router.xml @@ -0,0 +1,59 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>druid.service</name> + <value>druid/router</value> + <description>The druid.service name of router node.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.port</name> + <value>8888</value> + <description>The port on which the broker will accept connections.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.router.http.numConnections</name> + <value>20</value> + <description> + Size of connection pool for the router to connect to historical and real-time nodes. If there are more + queries than this number that all need to speak to the same node, then they will queue up. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.server.http.numThreads</name> + <value>50</value> + <description>Number of threads for HTTP requests.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>druid.router.tierToBrokerMap</name> + <value>{}</value> + <description> + Used to route queries for a certain tier of data to their appropriate broker. An ordered JSON map of + tiers to broker names. The priority of brokers is based on the ordering. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml new file mode 100644 index 0000000..c897f12 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml @@ -0,0 +1,249 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>DRUID</name> + <displayName>Druid</displayName> + <comment>A fast column-oriented distributed data store.</comment> + <version>0.9.2</version> + <components> + <component> + <name>DRUID_COORDINATOR</name> + <displayName>Druid Coordinator</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/coordinator.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>DRUID_OVERLORD</name> + <displayName>Druid Overlord</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/overlord.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>DRUID_HISTORICAL</name> + <displayName>Druid Historical</displayName> + <category>SLAVE</category> + <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/historical.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>DRUID_BROKER</name> + <displayName>Druid Broker</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/broker.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>DRUID_MIDDLEMANAGER</name> + <displayName>Druid MiddleManager</displayName> + <category>SLAVE</category> + <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/middlemanager.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>DRUID_ROUTER</name> + <displayName>Druid Router</displayName> + <category>SLAVE</category> + <cardinality>0+</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/router.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <themes> + <theme> + <fileName>theme.json</fileName> + <default>true</default> + </theme> + </themes> + + <osSpecifics> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>druid_${stack_version}</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>druid-${stack_version}</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>mysql</name> + <skipUpgrade>true</skipUpgrade> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>amazon2015,redhat6,debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>mysql-server</name> + <skipUpgrade>true</skipUpgrade> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>redhat7</osFamily> + <packages> + <package> + <name>mysql-community-release</name> + <skipUpgrade>true</skipUpgrade> + </package> + <package> + <name>mysql-community-server</name> + <skipUpgrade>true</skipUpgrade> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>suse11,suse12</osFamily> + <packages> + <package> + <name>mysql-client</name> + <skipUpgrade>true</skipUpgrade> + </package> + </packages> + </osSpecific> + </osSpecifics> + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + <requiredServices> + <service>ZOOKEEPER</service> + </requiredServices> + <configuration-dependencies> + <config-type>druid-common</config-type> + <config-type>druid-env</config-type> + <config-type>druid-coordinator</config-type> + <config-type>druid-overlord</config-type> + <config-type>druid-historical</config-type> + <config-type>druid-broker</config-type> + <config-type>druid-middlemanager</config-type> + <config-type>druid-log4j</config-type> + <config-type>druid-logrotate</config-type> + <config-type>druid-router</config-type> + <config-type>zoo.cfg</config-type> + <config-type>core-site</config-type> + <config-type>mapred-site</config-type> + <config-type>yarn-site</config-type> + <config-type>hdfs-site</config-type> + </configuration-dependencies> + + <quickLinksConfigurations> + <quickLinksConfiguration> + <fileName>quicklinks.json</fileName> + <default>true</default> + </quickLinksConfiguration> + </quickLinksConfigurations> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/broker.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/broker.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/broker.py new file mode 100644 index 0000000..bd170cb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/broker.py @@ -0,0 +1,28 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from druid_node import DruidBase + + +class DruidBroker(DruidBase): + def __init__(self): + DruidBase.__init__(self, nodeType="broker") + + +if __name__ == "__main__": + DruidBroker().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/coordinator.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/coordinator.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/coordinator.py new file mode 100644 index 0000000..a86fa40 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/coordinator.py @@ -0,0 +1,28 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from druid_node import DruidBase + + +class DruidCoordinator(DruidBase): + def __init__(self): + DruidBase.__init__(self, nodeType="coordinator") + + +if __name__ == "__main__": + DruidCoordinator().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py new file mode 100644 index 0000000..0478d17 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py @@ -0,0 +1,259 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.core.resources.system import Directory, Execute, File +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions import format +from resource_management.libraries.resources import XmlConfig +from resource_management.libraries.functions.show_logs import show_logs +from resource_management.core.logger import Logger + + +def druid(upgrade_type=None, nodeType=None): + import params + ensure_base_directories() + + # Environment Variables + File(format("{params.druid_conf_dir}/druid-env.sh"), + owner=params.druid_user, + content=InlineTemplate(params.druid_env_sh_template) + ) + + # common config + druid_common_config = mutable_config_dict(params.config['configurations']['druid-common']) + # User cannot override below configs + druid_common_config['druid.host'] = params.hostname + druid_common_config['druid.extensions.directory'] = params.druid_extensions_dir + druid_common_config['druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir + druid_common_config['druid.selectors.indexing.serviceName'] = params.config['configurations']['druid-overlord'][ + 'druid.service'] + druid_common_config['druid.selectors.coordinator.serviceName'] = \ + params.config['configurations']['druid-coordinator']['druid.service'] + + # delete the password and user if empty otherwiswe derby will fail. + if 'derby' == druid_common_config['druid.metadata.storage.type']: + del druid_common_config['druid.metadata.storage.connector.user'] + del druid_common_config['druid.metadata.storage.connector.password'] + + druid_env_config = mutable_config_dict(params.config['configurations']['druid-env']) + + PropertiesFile("common.runtime.properties", + dir=params.druid_common_conf_dir, + properties=druid_common_config, + owner=params.druid_user, + group=params.user_group, + ) + Logger.info("Created common.runtime.properties") + + File(format("{params.druid_common_conf_dir}/druid-log4j.xml"), + mode=0644, + owner=params.druid_user, + group=params.user_group, + content=InlineTemplate(params.log4j_props) + ) + Logger.info("Created log4j file") + + File("/etc/logrotate.d/druid", + mode=0644, + owner='root', + group='root', + content=InlineTemplate(params.logrotate_props) + ) + + Logger.info("Created log rotate file") + + # Write Hadoop Configs if configured + if 'core-site' in params.config['configurations']: + XmlConfig("core-site.xml", + conf_dir=params.druid_common_conf_dir, + configurations=params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner=params.druid_user, + group=params.user_group + ) + + if 'mapred-site' in params.config['configurations']: + XmlConfig("mapred-site.xml", + conf_dir=params.druid_common_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.druid_user, + group=params.user_group + ) + + if 'yarn-site' in params.config['configurations']: + XmlConfig("yarn-site.xml", + conf_dir=params.druid_common_conf_dir, + configurations=params.config['configurations']['yarn-site'], + configuration_attributes=params.config['configuration_attributes']['yarn-site'], + owner=params.druid_user, + group=params.user_group + ) + + if 'hdfs-site' in params.config['configurations']: + XmlConfig("hdfs-site.xml", + conf_dir=params.druid_common_conf_dir, + configurations=params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner=params.druid_user, + group=params.user_group + ) + + # node specific configs + for node_type in ['coordinator', 'overlord', 'historical', 'broker', 'middleManager', 'router']: + node_config_dir = format('{params.druid_conf_dir}/{node_type}') + node_type_lowercase = node_type.lower() + + # Write runtime.properties file + node_config = mutable_config_dict(params.config['configurations'][format('druid-{node_type_lowercase}')]) + PropertiesFile("runtime.properties", + dir=node_config_dir, + properties=node_config, + owner=params.druid_user, + group=params.user_group, + ) + Logger.info(format("Created druid-{node_type_lowercase} runtime.properties")) + + # Write jvm configs + File(format('{node_config_dir}/jvm.config'), + owner=params.druid_user, + group=params.user_group, + content=InlineTemplate( + "-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", + node_heap_memory=druid_env_config[format('druid.{node_type_lowercase}.jvm.heap.memory')], + log4j_config_file=format("{params.druid_common_conf_dir}/druid-log4j.xml"), + node_direct_memory=druid_env_config[ + format('druid.{node_type_lowercase}.jvm.direct.memory')], + node_jvm_opts=druid_env_config[format('druid.{node_type_lowercase}.jvm.opts')]) + ) + Logger.info(format("Created druid-{node_type_lowercase} jvm.config")) + + # All druid nodes have dependency on hdfs_client + ensure_hadoop_directories() + # Pull all required dependencies + pulldeps() + + +def mutable_config_dict(config): + rv = {} + for key, value in config.iteritems(): + rv[key] = value + return rv + + +def ensure_hadoop_directories(): + import params + if 'hdfs-site' not in params.config['configurations']: + # HDFS Not Installed nothing to do. + Logger.info("Skipping HDFS directory creation as HDFS not installed") + return + + druid_common_config = params.config['configurations']['druid-common'] + # final overlord config contains both common and overlord config + druid_middlemanager_config = params.config['configurations']['druid-middlemanager'] + + # If user is using HDFS as deep storage create HDFS Directory for storing segments + deep_storage = druid_common_config["druid.storage.type"] + storage_dir = druid_common_config["druid.storage.storageDirectory"] + + if deep_storage == 'hdfs': + # create the home dir for druid + params.HdfsResource(format("/user/{params.druid_user}"), + type="directory", + action="create_on_execute", + owner=params.druid_user, + recursive_chown=True, + recursive_chmod=True + ) + + # create the segment storage dir + create_hadoop_directory(storage_dir) + + # Create HadoopIndexTask hadoopWorkingPath + hadoop_working_path = druid_middlemanager_config['druid.indexer.task.hadoopWorkingPath'] + if hadoop_working_path is not None: + create_hadoop_directory(hadoop_working_path) + + # If HDFS is used for storing logs, create Index Task log directory + indexer_logs_type = druid_common_config['druid.indexer.logs.type'] + indexer_logs_directory = druid_common_config['druid.indexer.logs.directory'] + if indexer_logs_type == 'hdfs' and indexer_logs_directory is not None: + create_hadoop_directory(indexer_logs_directory) + + +def create_hadoop_directory(hadoop_dir): + import params + params.HdfsResource(hadoop_dir, + type="directory", + action="create_on_execute", + owner=params.druid_user, + mode=0755 + ) + Logger.info(format("Created Hadoop Directory [{hadoop_dir}]")) + + +def ensure_base_directories(): + import params + Directory( + [params.druid_log_dir, params.druid_pid_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir, + params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir, + params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir], + mode=0755, + cd_access='a', + owner=params.druid_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True, + ) + + +def get_daemon_cmd(params=None, node_type=None, command=None): + return format('source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}') + + +def getPid(params=None, nodeType=None): + return format('{params.druid_pid_dir}/{nodeType}.pid') + + +def pulldeps(): + import params + extensions_list = eval(params.druid_extensions) + extensions_string = '{0}'.format("-c ".join(extensions_list)) + if len(extensions_list) > 0: + try: + # Make sure druid user has permissions to write dependencies + Directory( + [params.druid_extensions_dir, params.druid_hadoop_dependencies_dir], + mode=0755, + cd_access='a', + owner=params.druid_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True, + ) + Execute(format( + "source {params.druid_conf_dir}/druid-env.sh ; java -classpath '{params.druid_home}/lib/*' -Ddruid.extensions.loadList=[] " + "-Ddruid.extensions.directory={params.druid_extensions_dir} -Ddruid.extensions.hadoopDependenciesDir={params.druid_hadoop_dependencies_dir} " + "io.druid.cli.Main tools pull-deps -c {extensions_string} --no-default-hadoop"), + user=params.druid_user + ) + Logger.info(format("Pull Dependencies Complete")) + except: + show_logs(params.druid_log_dir, params.druid_user) + raise http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py new file mode 100644 index 0000000..3b7f695 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py @@ -0,0 +1,85 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management import Script +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.check_process_status import check_process_status + +from resource_management.libraries.functions.show_logs import show_logs +from druid import druid, get_daemon_cmd, getPid + + +class DruidBase(Script): + def __init__(self, nodeType=None): + self.nodeType = nodeType + + def get_component_name(self): + node_type_lower = self.nodeType.lower() + return format("druid-{node_type_lower}") + + def install(self, env): + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + druid(upgrade_type=upgrade_type, nodeType=self.nodeType) + + def pre_upgrade_restart(self, env, upgrade_type=None): + return + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env, upgrade_type=upgrade_type) + daemon_cmd = get_daemon_cmd(params, self.nodeType, "start") + try: + Execute(daemon_cmd, + user=params.druid_user + ) + except: + show_logs(params.druid_log_dir, params.druid_user) + raise + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + daemon_cmd = get_daemon_cmd(params, self.nodeType, "stop") + try: + Execute(daemon_cmd, + user=params.druid_user + ) + except: + show_logs(params.druid_log_dir, params.druid_user) + raise + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = getPid(status_params, self.nodeType) + check_process_status(pid_file) + + def get_log_folder(self): + import params + return params.druid_log_dir + + def get_user(self): + import params + return params.druid_user http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py new file mode 100644 index 0000000..22390a6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py @@ -0,0 +1,28 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from druid_node import DruidBase + + +class DruidHistorical(DruidBase): + def __init__(self): + DruidBase.__init__(self, nodeType="historical") + + +if __name__ == "__main__": + DruidHistorical().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py new file mode 100644 index 0000000..20df89c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py @@ -0,0 +1,28 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from druid_node import DruidBase + + +class DruidMiddleManager(DruidBase): + def __init__(self): + DruidBase.__init__(self, nodeType="middleManager") + + +if __name__ == "__main__": + DruidMiddleManager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py new file mode 100644 index 0000000..e4d7fcc --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py @@ -0,0 +1,28 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from druid_node import DruidBase + + +class DruidOverlord(DruidBase): + def __init__(self): + DruidBase.__init__(self, nodeType="overlord") + + +if __name__ == "__main__": + DruidOverlord().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py new file mode 100644 index 0000000..00add3f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import format +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.functions.default import default + +import status_params + +# a map of the Ambari role to the component name +# for use with <stack-root>/current/<component> +SERVER_ROLE_DIRECTORY_MAP = { + 'DRUID_BROKER': 'druid-broker', + 'DRUID_COORDINATOR': 'druid-coordinator', + 'DRUID_HISTORICAL': 'druid-historical', + 'DRUID_MIDDLEMANAGER': 'druid-middlemanager', + 'DRUID_OVERLORD': 'druid-overlord', + 'DRUID_ROUTER': 'druid-router' +} + +# server configurations +config = Script.get_config() +stack_root = Script.get_stack_root() +tmp_dir = Script.get_tmp_dir() + +# stack version +stack_version = default("/commandParams/version", None) + +# default role to coordinator needed for service checks +component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "DRUID_COORDINATOR") + +hostname = config['hostname'] + +# default druid parameters +druid_home = format("{stack_root}/current/{component_directory}") +druid_conf_dir = format("{stack_root}/current/{component_directory}/conf") + +druid_common_conf_dir = druid_conf_dir + "/_common" +druid_coordinator_conf_dir = druid_conf_dir + "/coordinator" +druid_overlord_conf_dir = druid_conf_dir + "/overlord" +druid_broker_conf_dir = druid_conf_dir + "/broker" +druid_historical_conf_dir = druid_conf_dir + "/historical" +druid_middlemanager_conf_dir = druid_conf_dir + "/middleManager" +druid_router_conf_dir = druid_conf_dir + "/router" +druid_extensions_dir = druid_home + "/extensions" +druid_hadoop_dependencies_dir = druid_home + "/hadoop-dependencies" +druid_segment_infoDir = config['configurations']['druid-historical']['druid.segmentCache.infoDir'] +druid_user = config['configurations']['druid-env']['druid_user'] +druid_log_dir = config['configurations']['druid-env']['druid_log_dir'] +druid_classpath = config['configurations']['druid-env']['druid_classpath'] +druid_extensions = config['configurations']['druid-common']['druid.extensions.pullList'] + +# status params +druid_pid_dir = status_params.druid_pid_dir +user_group = config['configurations']['cluster-env']['user_group'] +java8_home = config['hostLevelParams']['java_home'] +druid_env_sh_template = config['configurations']['druid-env']['content'] + +# log4j params +log4j_props = config['configurations']['druid-log4j']['content'] +druid_log_level = config['configurations']['druid-log4j']['druid_log_level'] +metamx_log_level = config['configurations']['druid-log4j']['metamx_log_level'] +root_log_level = config['configurations']['druid-log4j']['root_log_level'] +logrotate_props = config['configurations']['druid-logrotate']['content'] + +# Metadata storage +metadata_storage_user = config['configurations']['druid-common']['druid.metadata.storage.connector.user'] +metadata_storage_password = config['configurations']['druid-common']['druid.metadata.storage.connector.password'] +metadata_storage_db_name = config['configurations']['druid-common']['database_name'] + +# HDFS +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', 'missing_principal').replace("_HOST", + hostname) +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] +dfs_type = default("/commandParams/dfs_type", "") + +# Kerberose +druid_principal_name = default('/configurations/druid-common/druid.hadoop.security.kerberos.principal', + 'missing_principal') +druid_user_keytab = default('/configurations/druid-common/druid.hadoop.security.kerberos.keytab', 'missing_keytab') + +import functools + +# create partial functions with common arguments for every HdfsResource call +# to create hdfs directory we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled=security_enabled, + keytab=hdfs_user_keytab, + kinit_path_local=kinit_path_local, + hadoop_bin_dir=hadoop_bin_dir, + hadoop_conf_dir=hadoop_conf_dir, + principal_name=hdfs_principal_name, + hdfs_site=hdfs_site, + default_fs=default_fs, + immutable_paths=get_not_managed_resources(), + dfs_type=dfs_type +) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py new file mode 100644 index 0000000..1731a2a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py @@ -0,0 +1,28 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from druid_node import DruidBase + + +class DruidRouter(DruidBase): + def __init__(self): + DruidBase.__init__(self, nodeType="router") + + +if __name__ == "__main__": + DruidRouter().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py new file mode 100644 index 0000000..139b727 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.format import format +from resource_management.core.resources.system import Execute + + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + self.checkComponent(params, "druid_coordinator", "druid-coordinator") + self.checkComponent(params, "druid_overlord", "druid-overlord") + + def checkComponent(self, params, component_name, config_name): + component_port = params.config['configurations'][format('{config_name}')]['druid.port'] + for component_host in params.config['clusterHostInfo'][format('{component_name}_hosts')]: + Execute(format( + "curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {component_host}:{component_port}/status | grep 200"), + tries=10, + try_sleep=3, + logoutput=True) + + +if __name__ == "__main__": + ServiceCheck().execute()
