http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py deleted file mode 100755 index ef9f6dd..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate - - -def properties_inline_template(configurations): - return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }} -{% endfor %}''', configurations_dict=configurations) - - -def properties_config(filename, configurations=None, conf_dir=None, - mode=None, owner=None, group=None, brokerid=None): - config_content = properties_inline_template(configurations) - File(format("{conf_dir}/{filename}"), content=config_content, owner=owner, - group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py deleted file mode 100755 index e84fb01..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from __future__ import print_function - -import subprocess -import sys - -from resource_management.core.resources.system import Execute -from resource_management.libraries.script import Script - - -class ServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - - doc = '{"name": "Ambari Smoke test"}' - index = "ambari_smoke_test" - - print("Running Elastic search service check", file=sys.stdout) - - # Make sure the service is actually up. We can live without everything allocated. - # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can - # also hit ES before things are green. - host = "localhost:9200" - Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host, - logoutput=True, - tries=6, - try_sleep=20 - ) - - # Put a document into a new index. - - Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True) - - # Retrieve the document. Use subprocess because we actually need the results here. - cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index) - proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() - response_retrieve = stdout - print("Retrieval response is: %s" % response_retrieve) - expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \ - % (index, doc) - - # Delete the index - cmd_delete = "curl -XDELETE '%s/%s'" % (host, index) - proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (stdout, stderr) = proc.communicate() - response_delete = stdout - print("Delete index response is: %s" % response_retrieve) - expected_delete = '{"acknowledged":true}' - - if (expected_retrieve == response_retrieve) and (expected_delete == response_delete): - print("Smoke test able to communicate with Elasticsearch") - else: - print("Elasticsearch service unable to retrieve document.") - sys.exit(1) - - exit(0) - - -if __name__ == "__main__": - ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py deleted file mode 100755 index 15a5cf5..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate -from resource_management.core.source import Template -from resource_management.core.resources import User - -def slave(): - import params - - User(params.elastic_user, action = "create", groups = params.elastic_group) - params.path_data = params.path_data.replace('"', '') - data_path = params.path_data.replace(' ', '').split(',') - data_path[:] = [x.replace('"', '') for x in data_path] - - directories = [params.log_dir, params.pid_dir, params.conf_dir] - directories = directories + data_path - - Directory(directories, - create_parents=True, - mode=0755, - owner=params.elastic_user, - group=params.elastic_group, - cd_access="a" - ) - - File("{0}/elastic-env.sh".format(params.conf_dir), - owner=params.elastic_user, - content=InlineTemplate(params.elastic_env_sh_template) - ) - - configurations = params.config['configurations']['elastic-site'] - - File("{0}/elasticsearch.yml".format(params.conf_dir), - content=Template( - "elasticsearch.slave.yaml.j2", - configurations=configurations), - owner=params.elastic_user, - group=params.elastic_group - ) - - print "Master sysconfig: /etc/sysconfig/elasticsearch" - File(format("/etc/sysconfig/elasticsearch"), - owner="root", - group="root", - content=InlineTemplate(params.sysconfig_template) - ) http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py deleted file mode 100755 index 9cfb5cf..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.script import Script - -config = Script.get_config() - -elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] -elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid") http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 deleted file mode 100755 index f0d2a8f..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 +++ /dev/null @@ -1,85 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -cluster: - name: {{cluster_name}} - routing: - allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} - allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} - allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} - allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} - -discovery: - zen: - ping: - multicast: - enabled: {{ discovery_zen_ping_multicast_enabled }} - unicast: - hosts: {{zen_discovery_ping_unicast_hosts}} - -node: - data: {{ masters_also_are_datanodes }} - master: true - name: {{hostname}} -path: - data: {{path_data}} - -http.cors.enabled: {{http_cors_enabled}} - -port: {{http_port}} - -transport: - tcp: - port: {{transport_tcp_port}} - -gateway: - recover_after_data_nodes: {{gateway_recover_after_data_nodes}} - recover_after_time: {{recover_after_time}} - expected_data_nodes: {{expected_data_nodes}} - -index: - number_of_shards: {{index_number_of_shards}} - merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} - translog.flush_threshold_size: {{index_translog_flush_threshold_size}} - refresh_interval: {{index_refresh_interval}} - number_of_replicas: {{index_number_of_replicas}} - -indices: - memory: - index_buffer_size: {{indices_memory_index_buffer_size}} - store.throttle.type: {{indices_memory_index_store_throttle_type}} - fielddata: - cache.size: {{indices_fielddata_cache_size}} - cluster: - send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} - -bootstrap.mlockall: {{bootstrap_mlockall}} - -threadpool: - bulk: - queue_size: {{threadpool_bulk_queue_size}} - index: - queue_size: {{threadpool_index_queue_size}} - -discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} -discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} -discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} -discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} - -network.host: {{network_host}} -network.publish_host: {{network_publish_host}} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 deleted file mode 100755 index 7d2d0cf..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 +++ /dev/null @@ -1,85 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -cluster: - name: {{cluster_name}} - routing: - allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} - allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} - allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} - allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} - -discovery: - zen: - ping: - multicast: - enabled: {{discovery_zen_ping_multicast_enabled}} - unicast: - hosts: {{zen_discovery_ping_unicast_hosts}} - -node: - data: true - master: false - name: {{hostname}} -path: - data: {{path_data}} - -http.cors.enabled: {{http_cors_enabled}} - -port: {{http_port}} - -transport: - tcp: - port: {{transport_tcp_port}} - -gateway: - recover_after_data_nodes: {{gateway_recover_after_data_nodes}} - recover_after_time: {{recover_after_time}} - expected_data_nodes: {{expected_data_nodes}} - -index: - number_of_shards: {{index_number_of_shards}} - merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}} - translog.flush_threshold_size: {{index_translog_flush_threshold_size}} - refresh_interval: {{index_refresh_interval}} - number_of_replicas: {{index_number_of_replicas}} - -indices: - memory: - index_buffer_size: {{indices_memory_index_buffer_size}} - store.throttle.type: {{indices_memory_index_store_throttle_type}} - fielddata: - cache.size: {{indices_fielddata_cache_size}} - cluster: - send_refresh_mapping: {{indices_cluster_send_refresh_mapping}} - -bootstrap.mlockall: {{bootstrap_mlockall}} - -threadpool: - bulk: - queue_size: {{threadpool_bulk_queue_size}} - index: - queue_size: {{threadpool_index_queue_size}} - -discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} -discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} -discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} -discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} - -network.host: {{network_host}} -network.publish_host: {{network_publish_host}} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json deleted file mode 100644 index 909828b..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"HTTP_ONLY" - }, - - "links": [ - { - "name": "es_health_link", - "label": "Elasticsearch Health", - "requires_user_name": "false", - "component_name": "ES_MASTER", - "url":"%@://%@:%@/_cat/health?v", - "port":{ - "http_property": "http_port", - "http_default_port": "9200", - "https_property": "http_port", - "https_default_port": "9200", - "regex": "^(\\d+)", - "site": "elastic-site" - } - }, - { - "name": "es_indices_link", - "label": "Elasticsearch Indexes", - "requires_user_name": "false", - "component_name": "ES_MASTER", - "url":"%@://%@:%@/_cat/indices?v", - "port":{ - "http_property": "http_port", - "http_default_port": "9200", - "https_property": "http_port", - "https_default_port": "9200", - "regex": "^(\\d+)", - "site": "elastic-site" - } - } - ] - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json deleted file mode 100755 index 130d018..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "_comment" : "Record format:", - "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", - "general_deps" : { - "_comment" : "dependencies for all cases", - "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"] - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml new file mode 100755 index 0000000..9e4f8ad --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml @@ -0,0 +1,86 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>elastic_user</name> + <value>elasticsearch</value> + <property-type>USER</property-type> + <description>Service user for Elasticsearch</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>elastic_group</name> + <value>elasticsearch</value> + <property-type>GROUP</property-type> + <description>Service group for Elasticsearch</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>elastic_log_dir</name> + <value>/var/log/elasticsearch</value> + <description>Log directory for elastic</description> + </property> + <property> + <name>elastic_pid_dir</name> + <value>/var/run/elasticsearch</value> + <description>The directory for pid files</description> + </property> + <!-- elasticsearch-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for elastic-env.sh file</description> + <value> +#!/bin/bash + +# Set ELASTICSEARCH specific environment variables here. + +# The java implementation to use. +export JAVA_HOME={{java64_home}} +export PATH=$PATH:$JAVA_HOME/bin + </value> + </property> + <property> + <name>elastic_user_nofile_limit</name> + <value>65536</value> + <description>Max open file limit for Elasticsearch user.</description> + </property> + <property> + <name>elastic_user_nproc_limit</name> + <value>2048</value> + <description>Max number of processes for Elasticsearch user.</description> + </property> + <property> + <name>elastic_user_memlock_soft_limit</name> + <value>unlimited</value> + <description>Max locked-in memory address space (soft memlock limit).</description> + </property> + <property> + <name>elastic_user_memlock_hard_limit</name> + <value>unlimited</value> + <description>Max locked-in memory address space (hard memlock limit).</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml new file mode 100755 index 0000000..34df1e4 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml @@ -0,0 +1,198 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- Elastic search Configurations --> + +<configuration supports_final="true"> + <!-- Configurations --> + <property> + <name>cluster_name</name> + <value>metron</value> + <description>Elasticsearch Cluster Name identifies your Elasticsearch subsystem</description> + </property> + <property> + <name>masters_also_are_datanodes</name> + <value>"false"</value> + <description>ES Masters and Slaves cannot be installed on the same nodes. Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes.</description> + <value-attributes> + <type>string</type> + </value-attributes> + </property> + <property> + <name>zen_discovery_ping_unicast_hosts</name> + <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.--> + <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.--> + <value></value> + <description>Unicast discovery list of hosts to act as gossip routers, comma-separated list with square brackets: [ eshost1, eshost2 ]</description> + </property> + <property> + <name>index_number_of_shards</name> + <value>4</value> + <description>Set the number of shards (splits) of an index. Changes are not effective after index creation. Usually set to 1 for single-node install.</description> + </property> + <property> + <name>index_number_of_replicas</name> + <value>2</value> + <description>Set the number of replicas (copies in addition to the first) of an index. Usually set to 0 for single-node install.</description> + </property> + <property> + <name>path_data</name> + <value>"/opt/lmm/es_data"</value> + <description>Comma-separated list of directories where to store index data allocated for each node: "/mnt/first","/mnt/second". Number of paths should relate to number of shards, and preferably should be on separate physical volumes.</description> + </property> + <property> + <name>http_cors_enabled</name> + <value>"false"</value> + <description>Enable or disable cross-origin resource sharing, i.e. whether a browser on another origin can do requests to Elasticsearch. Defaults to false.</description> + <value-attributes> + <type>string</type> + </value-attributes> + </property> + <property> + <name>http_port</name> + <value>9200-9300</value> + <description>Set a custom port to listen for HTTP traffic</description> + </property> + <property> + <name>transport_tcp_port</name> + <value>9300-9400</value> + <description>Set a custom port for the node to node communication</description> + </property> + <!-- Multi-node Discovery --> + <property> + <name>discovery_zen_ping_timeout</name> + <value>3s</value> + <description>Wait for ping responses for master discovery</description> + </property> + <property> + <name>discovery_zen_fd_ping_interval</name> + <value>15s</value> + <description>Wait for ping for cluster discovery</description> + </property> + <property> + <name>discovery_zen_fd_ping_timeout</name> + <value>60s</value> + <description>Wait for ping for cluster discovery</description> + </property> + <property> + <name>discovery_zen_fd_ping_retries</name> + <value>5</value> + <description>Number of ping retries before blacklisting</description> + </property> + <!-- Gateway --> + <property> + <name>gateway_recover_after_data_nodes</name> + <value>3</value> + <description>Recover as long as this many data or master nodes have joined the cluster.</description> + </property> + <property> + <name>recover_after_time</name> + <value>15m</value> + <description>recover_after_time</description> + </property> + <property> + <name>expected_data_nodes</name> + <value>0</value> + <description>expected_data_nodes</description> + </property> + <!-- Index --> + <property> + <name>index_merge_scheduler_max_thread_count</name> + <value>5</value> + <description>index.merge.scheduler.max_thread_count</description> + </property> + <property> + <name>indices_memory_index_store_throttle_type</name> + <value>none</value> + <description>index_store_throttle_type</description> + </property> + <property> + <name>index_refresh_interval</name> + <value>1s</value> + <description>index refresh interval</description> + </property> + <property> + <name>index_translog_flush_threshold_size</name> + <value>5g</value> + <description>index_translog_flush_threshold_size</description> + </property> + <property> + <name>indices_memory_index_buffer_size</name> + <value>10%</value> + <description>Percentage of heap used for write buffers</description> + </property> + <property> + <name>bootstrap_memory_lock</name> + <value>true</value> + <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description> + </property> + <property> + <name>threadpool_bulk_queue_size</name> + <value>3000</value> + <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description> + </property> + <property> + <name>threadpool_index_queue_size</name> + <value>1000</value> + <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description> + </property> + <property> + <name>indices_cluster_send_refresh_mapping</name> + <value>false</value> + <description>In order to make the index request more efficient, we have set this property on our data nodes</description> + </property> + <property> + <name>indices_fielddata_cache_size</name> + <value>25%</value> + <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description> + </property> + <property> + <name>cluster_routing_allocation_disk_watermark_high</name> + <value>0.99</value> + <description>Property used when multiple drives are used to understand max thresholds</description> + </property> + <property> + <name>cluster_routing_allocation_disk_threshold_enabled</name> + <value>true</value> + <description>Property used when multiple drives are used to understand if thresholding is active</description> + </property> + <property> + <name>cluster_routing_allocation_disk_watermark_low</name> + <value>.97</value> + <description>Property used when multiple drives are used to understand min thresholds</description> + </property> + <property> + <name>cluster_routing_allocation_node_concurrent_recoveries</name> + <value>4</value> + <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description> + </property> + <property> + <name>network_host</name> + <value>[ _local_, _site_ ]</value> + <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> + </property> + <property> + <name>network_publish_host</name> + <value>[]</value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml new file mode 100755 index 0000000..d6db027 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml @@ -0,0 +1,102 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>elastic_home</name> + <value>/usr/share/elasticsearch/</value> + <description>Elasticsearch Home Directory</description> + </property> + <property> + <name>data_dir</name> + <value>/var/lib/elasticsearch/</value> + <description>Elasticsearch Data Directory</description> + </property> + <property> + <name>work_dir</name> + <value>/tmp/elasticsearch/</value> + <description>Elasticsearch Work Directory</description> + </property> + <property> + <name>conf_dir</name> + <value>/etc/elasticsearch/</value> + <description>Elasticsearch Configuration Directory</description> + </property> + <property> + <name>heap_size</name> + <value>512m</value> + <description>Heap size</description> + </property> + <property> + <name>max_open_files</name> + <value>65536</value> + <description>Maximum number of open files</description> + </property> + <property> + <name>max_map_count</name> + <value>262144</value> + <description>Maximum number of memory map areas for process</description> + </property> + + <!--/etc/sysconfig/elasticsearch--> + <property> + <name>content</name> + <description>This is the jinja template for elastic sysconfig file (/etc/sysconfig/elasticsearch)</description> + <value> +# Directory where the Elasticsearch binary distribution resides +ES_HOME={{elastic_home}} + +# Maximum number of open files +MAX_OPEN_FILES={{max_open_files}} + +# Maximum number of VMA (Virtual Memory Areas) a process can own +MAX_MAP_COUNT={{max_map_count}} + +# Elasticsearch log directory +LOG_DIR={{log_dir}} + +# Elasticsearch data directory +DATA_DIR={{data_dir}} + +# Elasticsearch work directory +WORK_DIR={{work_dir}} + +# Elasticsearch conf directory +CONF_DIR={{conf_dir}} + +# User to run as, change this to a specific elasticsearch user if possible +# Also make sure, this user can write into the log directories in case you change them +# This setting only works for the init script, but has to be configured separately for systemd startup +ES_USER={{elastic_user}} + +# Elasticsearch pid directory +PID_DIR={{pid_dir}} + +# JAVA_HOME must be provided here for OS that use systemd service launch +JAVA_HOME={{java64_home}} + +# Additional Java OPTS +ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \ +-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \ +-XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8 \ +-Xms{{heap_size}} -Xmx{{heap_size}}" + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml new file mode 100755 index 0000000..3783d31 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml @@ -0,0 +1,79 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ELASTICSEARCH</name> + <displayName>Elasticsearch</displayName> + <comment>Indexing and Search</comment> + <version>5.6.2</version> + <components> + <component> + <name>ES_MASTER</name> + <displayName>Elasticsearch Master</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/elastic_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>ES_SLAVE</name> + <displayName>Elasticsearch Data Node</displayName> + <category>SLAVE</category> + <cardinality>0+</cardinality> + <commandScript> + <script>scripts/elastic_slave.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>elasticsearch-5.6.2</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + <configuration-dependencies> + <config-type>elastic-env</config-type> + <config-type>elastic-site</config-type> + <config-type>elastic-sysconfig</config-type> + </configuration-dependencies> + <restartRequiredAfterChange>true</restartRequiredAfterChange> + <quickLinksConfigurations> + <quickLinksConfiguration> + <fileName>quicklinks.json</fileName> + <default>true</default> + </quickLinksConfiguration> + </quickLinksConfigurations> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py new file mode 100755 index 0000000..e27e8bf --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.core.resources import User +from resource_management.core.logger import Logger +from resource_management.libraries.functions import format as ambari_format + +def elastic(): + import params + + Logger.info("Creating user: {0}:{1}".format(params.elastic_user, params.elastic_group)) + User(params.elastic_user, action = "create", groups = params.elastic_group) + + params.path_data = params.path_data.replace('"', '') + data_path = params.path_data.replace(' ', '').split(',') + data_path[:] = [x.replace('"', '') for x in data_path] + directories = [params.log_dir, params.pid_dir, params.conf_dir] + directories = directories + data_path + ["{0}/scripts".format(params.conf_dir)] + + Logger.info("Creating directories: {0}".format(directories)) + Directory(directories, + create_parents=True, + mode=0755, + owner=params.elastic_user, + group=params.elastic_group + ) + + Logger.info("Master env: ""{0}/elastic-env.sh".format(params.conf_dir)) + File("{0}/elastic-env.sh".format(params.conf_dir), + owner=params.elastic_user, + group=params.elastic_group, + content=InlineTemplate(params.elastic_env_sh_template) + ) + + configurations = params.config['configurations']['elastic-site'] + Logger.info("Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir)) + File("{0}/elasticsearch.yml".format(params.conf_dir), + content=Template( + "elasticsearch.master.yaml.j2", + configurations=configurations), + owner=params.elastic_user, + group=params.elastic_group + ) + + Logger.info("Master sysconfig: /etc/sysconfig/elasticsearch") + File("/etc/sysconfig/elasticsearch", + owner="root", + group="root", + content=InlineTemplate(params.sysconfig_template) + ) + + # in some OS this folder may not exist, so create it + Logger.info("Ensure PAM limits directory exists: {0}".format(params.limits_conf_dir)) + Directory(params.limits_conf_dir, + create_parents=True, + owner='root', + group='root' + ) + + Logger.info("Master PAM limits: {0}".format(params.limits_conf_file)) + File(params.limits_conf_file, + content=Template('elasticsearch_limits.conf.j2'), + owner="root", + group="root" + ) http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py new file mode 100755 index 0000000..c3f089a --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py @@ -0,0 +1,72 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script +from resource_management.core.logger import Logger + +from elastic import elastic + + +class Elasticsearch(Script): + def install(self, env): + import params + env.set_params(params) + Logger.info('Install Elasticsearch master node') + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + Logger.info('Configure Elasticsearch master node') + elastic() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + Logger.info('Stop Elasticsearch master node') + stop_cmd = "service elasticsearch stop" + Execute(stop_cmd) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + Logger.info('Start Elasticsearch master node') + self.configure(env) + start_cmd = "service elasticsearch start" + Execute(start_cmd) + + def status(self, env): + import params + env.set_params(params) + Logger.info('Check status of Elasticsearch master node') + status_cmd = "service elasticsearch status" + Execute(status_cmd) + + def restart(self, env): + import params + env.set_params(params) + self.configure(env) + Logger.info('Restart Elasticsearch master node') + restart_cmd = "service elasticsearch restart" + Execute(restart_cmd) + + +if __name__ == "__main__": + Elasticsearch().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py new file mode 100755 index 0000000..8aaee75 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py @@ -0,0 +1,72 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script +from resource_management.core.logger import Logger + +from slave import slave + + +class Elasticsearch(Script): + def install(self, env): + import params + env.set_params(params) + Logger.info('Install Elasticsearch data node') + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + Logger.info('Configure Elasticsearch data node') + slave() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + Logger.info('Stop Elasticsearch data node') + stop_cmd = "service elasticsearch stop" + Execute(stop_cmd) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + Logger.info('Start Elasticsearch data node') + start_cmd = "service elasticsearch start" + Execute(start_cmd) + + def status(self, env): + import params + env.set_params(params) + Logger.info('Check status of Elasticsearch data node') + status_cmd = "service elasticsearch status" + Execute(status_cmd) + + def restart(self, env): + import params + env.set_params(params) + self.configure(env) + Logger.info('Restart Elasticsearch data node') + restart_cmd = "service elasticsearch restart" + Execute(restart_cmd) + + +if __name__ == "__main__": + Elasticsearch().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py new file mode 100755 index 0000000..4adcf43 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script + +def yamlify_variables(var) : + if isinstance(var, type(True)): + return str(var).lower() + else: + return var + +# server configurations +config = Script.get_config() + +masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes'] +elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] +data_dir = config['configurations']['elastic-sysconfig']['data_dir'] +work_dir = config['configurations']['elastic-sysconfig']['work_dir'] +conf_dir = config['configurations']['elastic-sysconfig']['conf_dir'] +heap_size = config['configurations']['elastic-sysconfig']['heap_size'] +max_open_files = config['configurations']['elastic-sysconfig']['max_open_files'] +max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] + +elastic_user = config['configurations']['elastic-env']['elastic_user'] +elastic_group = config['configurations']['elastic-env']['elastic_group'] +log_dir = config['configurations']['elastic-env']['elastic_log_dir'] +pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] + +hostname = config['hostname'] +java64_home = config['hostLevelParams']['java_home'] +elastic_env_sh_template = config['configurations']['elastic-env']['content'] +sysconfig_template = config['configurations']['elastic-sysconfig']['content'] + +cluster_name = config['configurations']['elastic-site']['cluster_name'] +zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts'] + +path_data = config['configurations']['elastic-site']['path_data'] +http_cors_enabled = config['configurations']['elastic-site']['http_cors_enabled'] +http_port = config['configurations']['elastic-site']['http_port'] +transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port'] + +recover_after_time = config['configurations']['elastic-site']['recover_after_time'] +gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes'] +expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes'] +index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count'] +index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size'] +index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval'] +indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type'] +index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] +index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] +indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] +bootstrap_memory_lock = yamlify_variables(config['configurations']['elastic-site']['bootstrap_memory_lock']) +threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] +cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] +cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low'] +cluster_routing_allocation_disk_threshold_enabled = yamlify_variables(config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled']) +cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high'] +indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size'] +indices_cluster_send_refresh_mapping = yamlify_variables(config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping']) +threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size'] + +discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout'] +discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval'] +discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout'] +discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries'] + +network_host = config['configurations']['elastic-site']['network_host'] +network_publish_host = config['configurations']['elastic-site']['network_publish_host'] + +limits_conf_dir = "/etc/security/limits.d" +limits_conf_file = limits_conf_dir + "/elasticsearch.conf" +elastic_user_nofile_limit = config['configurations']['elastic-env']['elastic_user_nofile_limit'] +elastic_user_nproc_limit = config['configurations']['elastic-env']['elastic_user_nproc_limit'] +elastic_user_memlock_soft_limit = config['configurations']['elastic-env']['elastic_user_memlock_soft_limit'] +elastic_user_memlock_hard_limit = config['configurations']['elastic-env']['elastic_user_memlock_hard_limit'] http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py new file mode 100755 index 0000000..ef9f6dd --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/properties_config.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate + + +def properties_inline_template(configurations): + return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }} +{% endfor %}''', configurations_dict=configurations) + + +def properties_config(filename, configurations=None, conf_dir=None, + mode=None, owner=None, group=None, brokerid=None): + config_content = properties_inline_template(configurations) + File(format("{conf_dir}/{filename}"), content=config_content, owner=owner, + group=group, mode=mode) http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py new file mode 100755 index 0000000..d59954f --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import subprocess +import sys + +from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script +from resource_management.core.logger import Logger + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + doc = '{"name": "Ambari Smoke test"}' + index = "ambari_smoke_test" + + Logger.info("Running Elastic search service check", file=sys.stdout) + + # Make sure the service is actually up. We can live without everything allocated. + # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can + # also hit ES before things are green. + host = "localhost:9200" + Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host, + logoutput=True, + tries=6, + try_sleep=20 + ) + + # Put a document into a new index. + Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True) + + # Retrieve the document. Use subprocess because we actually need the results here. + cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index) + proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + response_retrieve = stdout + Logger.info("Retrieval response is: %s" % response_retrieve) + expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \ + % (index, doc) + + # Delete the index + cmd_delete = "curl -XDELETE '%s/%s'" % (host, index) + proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + (stdout, stderr) = proc.communicate() + response_delete = stdout + Logger.info("Delete index response is: %s" % response_retrieve) + expected_delete = '{"acknowledged":true}' + + if (expected_retrieve == response_retrieve) and (expected_delete == response_delete): + Logger.info("Smoke test able to communicate with Elasticsearch") + else: + Logger.info("Elasticsearch service unable to retrieve document.") + sys.exit(1) + + exit(0) + + +if __name__ == "__main__": + ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py new file mode 100755 index 0000000..a19989e --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.core.resources import User +from resource_management.core.logger import Logger + +def slave(): + import params + + Logger.info("Creating user: {0}:{1}".format(params.elastic_user, params.elastic_group)) + User(params.elastic_user, action = "create", groups = params.elastic_group) + + params.path_data = params.path_data.replace('"', '') + data_path = params.path_data.replace(' ', '').split(',') + data_path[:] = [x.replace('"', '') for x in data_path] + directories = [params.log_dir, params.pid_dir, params.conf_dir] + directories = directories + data_path + + Logger.info("Creating directories: {0}".format(directories)) + Directory(directories, + create_parents=True, + mode=0755, + owner=params.elastic_user, + group=params.elastic_group, + cd_access="a" + ) + + File("{0}/elastic-env.sh".format(params.conf_dir), + owner=params.elastic_user, + content=InlineTemplate(params.elastic_env_sh_template) + ) + + elastic_site = params.config['configurations']['elastic-site'] + path = "{0}/elasticsearch.yml".format(params.conf_dir) + Logger.info("Creating ES slave configuration.") + File(path, + content=Template( + "elasticsearch.slave.yaml.j2", + configurations=elastic_site), + owner=params.elastic_user, + group=params.elastic_group + ) + + Logger.info("Slave sysconfig: /etc/sysconfig/elasticsearch") + File(format("/etc/sysconfig/elasticsearch"), + owner="root", + group="root", + content=InlineTemplate(params.sysconfig_template) + ) + + elastic_env = params.config['configurations']['elastic-env'] + Logger.info("Slave PAM limits: {0}".format(params.limits_conf_file)) + File(params.limits_conf_file, + content=Template( + 'elasticsearch_limits.conf.j2', + configurations=elastic_env), + owner="root", + group="root" + ) http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py new file mode 100755 index 0000000..9cfb5cf --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script + +config = Script.get_config() + +elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] +elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid") http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 new file mode 100755 index 0000000..8e20ba2 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.master.yaml.j2 @@ -0,0 +1,77 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +cluster: + name: {{cluster_name}} + routing: + allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} + allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} + allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} + allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} + +discovery: + zen: + ping: + unicast: + hosts: {{zen_discovery_ping_unicast_hosts}} + +node: + data: {{ masters_also_are_datanodes }} + master: true + name: {{hostname}} +path: + data: {{path_data}} + +http: + port: {{http_port}} + cors.enabled: {{http_cors_enabled}} + + +transport: + tcp: + port: {{transport_tcp_port}} + +gateway: + recover_after_data_nodes: {{gateway_recover_after_data_nodes}} + recover_after_time: {{recover_after_time}} + expected_data_nodes: {{expected_data_nodes}} +# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html +indices: + store.throttle.type: {{indices_memory_index_store_throttle_type}} + memory: + index_buffer_size: {{indices_memory_index_buffer_size}} + fielddata: + cache.size: {{indices_fielddata_cache_size}} + +bootstrap: + memory_lock: {{bootstrap_memory_lock}} + system_call_filter: false + +thread_pool: + bulk: + queue_size: {{threadpool_bulk_queue_size}} + index: + queue_size: {{threadpool_index_queue_size}} + +discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} +discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} +discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} +discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} + +network.host: {{network_host}} +network.publish_host: {{network_publish_host}} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 new file mode 100755 index 0000000..6bf8399 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch.slave.yaml.j2 @@ -0,0 +1,78 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +cluster: + name: {{cluster_name}} + routing: + allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}} + allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}} + allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}} + allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}} + +discovery: + zen: + ping: + unicast: + hosts: {{zen_discovery_ping_unicast_hosts}} + +node: + data: true + master: false + name: {{hostname}} +path: + data: {{path_data}} + +http: + port: {{http_port}} + cors.enabled: {{http_cors_enabled}} + + +transport: + tcp: + port: {{transport_tcp_port}} + +gateway: + recover_after_data_nodes: {{gateway_recover_after_data_nodes}} + recover_after_time: {{recover_after_time}} + expected_data_nodes: {{expected_data_nodes}} + +# https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html +indices: + store.throttle.type: {{indices_memory_index_store_throttle_type}} + memory: + index_buffer_size: {{indices_memory_index_buffer_size}} + fielddata: + cache.size: {{indices_fielddata_cache_size}} + +bootstrap: + memory_lock: {{bootstrap_memory_lock}} + system_call_filter: false + +thread_pool: + bulk: + queue_size: {{threadpool_bulk_queue_size}} + index: + queue_size: {{threadpool_index_queue_size}} + +discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}} +discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} +discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} +discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} + +network.host: {{network_host}} +network.publish_host: {{network_publish_host}} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 new file mode 100644 index 0000000..99f72e1 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/templates/elasticsearch_limits.conf.j2 @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{elastic_user}} - nproc {{elastic_user_nproc_limit}} +{{elastic_user}} - nofile {{elastic_user_nofile_limit}} +{{elastic_user}} soft memlock {{elastic_user_memlock_soft_limit}} +{{elastic_user}} hard memlock {{elastic_user_memlock_hard_limit}} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json new file mode 100644 index 0000000..909828b --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/quicklinks/quicklinks.json @@ -0,0 +1,43 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "es_health_link", + "label": "Elasticsearch Health", + "requires_user_name": "false", + "component_name": "ES_MASTER", + "url":"%@://%@:%@/_cat/health?v", + "port":{ + "http_property": "http_port", + "http_default_port": "9200", + "https_property": "http_port", + "https_default_port": "9200", + "regex": "^(\\d+)", + "site": "elastic-site" + } + }, + { + "name": "es_indices_link", + "label": "Elasticsearch Indexes", + "requires_user_name": "false", + "component_name": "ES_MASTER", + "url":"%@://%@:%@/_cat/indices?v", + "port":{ + "http_property": "http_port", + "http_default_port": "9200", + "https_property": "http_port", + "https_default_port": "9200", + "regex": "^(\\d+)", + "site": "elastic-site" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json new file mode 100755 index 0000000..130d018 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/role_command_order.json @@ -0,0 +1,8 @@ +{ + "_comment" : "Record format:", + "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]", + "general_deps" : { + "_comment" : "dependencies for all cases", + "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"] + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml deleted file mode 100755 index 0adf32a..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml +++ /dev/null @@ -1,67 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration> - <property> - <name>kibana_user</name> - <value>kibana</value> - <property-type>USER</property-type> - <description>Service User for Kibana</description> - <value-attributes> - <type>user</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>kabana_group</name> - <value>kibana</value> - <property-type>GROUP</property-type> - <description>Service Group for Kibana</description> - <value-attributes> - <type>user</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property require-input="true"> - <name>kibana_log_dir</name> - <value>/var/log/kibana</value> - <description>Log directory for Kibana</description> - </property> - <property require-input="true"> - <name>kibana_pid_dir</name> - <value>/var/run/kibana</value> - <description>PID directory for Kibana</description> - </property> - <property require-input="true"> - <name>kibana_es_url</name> - <value></value> - <description>The Elasticsearch instance to use for all your queries. (http://eshost:9200)</description> - </property> - <property require-input="true"> - <name>kibana_server_port</name> - <value>5000</value> - <description>Kibana back end server port to use.</description> - </property> - <property require-input="true"> - <name>kibana_default_application</name> - <value>default</value> - <description>The default application to load.</description> - </property> -</configuration>
