METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is the indexer (nickwallen) closes apache/metron#1056
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed4dfb93 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed4dfb93 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed4dfb93 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: ed4dfb93641c9c9a84b61b5daddbebd962b3b20c Parents: 2bf6650 Author: nickwallen <[email protected]> Authored: Thu Jun 21 11:28:30 2018 -0400 Committer: nickallen <[email protected]> Committed: Thu Jun 21 11:28:30 2018 -0400 ---------------------------------------------------------------------- .../METRON/CURRENT/configuration/metron-env.xml | 32 ++++-- .../package/scripts/indexing_commands.py | 4 +- .../CURRENT/package/scripts/indexing_master.py | 10 +- .../CURRENT/package/scripts/metron_service.py | 101 ++++++++++++++----- 4 files changed, 113 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 0b64f3a..cdef7cf 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -81,30 +81,54 @@ <value>metron</value> <description>Name of Elasticsearch Cluster</description> <display-name>Elasticsearch Cluster Name</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> - <property require-input="true"> + <property> <name>es_hosts</name> <value></value> <description>Comma delimited list of Elasticsearch Master Hosts: eshost1,eshost2</description> <display-name>Elasticsearch Hosts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> <property> <name>es_binary_port</name> <value>9300</value> <description>Elasticsearch binary port. (9300)</description> <display-name>Elasticsearch Binary Port</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> <property> <name>es_http_port</name> <value>9200</value> <description>Elasticsearch HTTP port. (9200)</description> <display-name>Elasticsearch HTTP port</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>es_date_format</name> + <description>Elasticsearch Date Format</description> + <value>yyyy.MM.dd.HH</value> + <display-name>Elasticsearch Date Format</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> <property> <name>solr_zookeeper_url</name> <value>{{zookeeper_quorum}}</value> <description>Comma delimited list of Zookeeper Urls: zkhost1:2181,zkhost1:2181</description> <display-name>Solr Zookeeper Urls</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> <property require-input = "true"> <name>storm_rest_addr</name> @@ -137,10 +161,4 @@ <empty-value-valid>true</empty-value-valid> </value-attributes> </property> - <property> - <name>es_date_format</name> - <description>Elasticsearch Date Format</description> - <value>yyyy.MM.dd.HH</value> - <display-name>Elasticsearch Date Format</display-name> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 69e980b..4802add 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -30,7 +30,6 @@ from resource_management.libraries.functions.format import format import metron_service import metron_security - # Wrap major operations and functionality in this class class IndexingCommands: __params = None @@ -394,7 +393,6 @@ class IndexingCommands: is_random_access_running = topologies[self.__random_access_indexing_topology] in ['ACTIVE', 'REBALANCING'] return is_random_access_running - def is_topology_active(self, env): return self.is_batch_topology_active(env) and self.is_random_access_topology_active(env) @@ -403,6 +401,8 @@ class IndexingCommands: Performs a service check for Indexing. :param env: Environment """ + metron_service.check_indexer_parameters() + Logger.info('Checking Kafka topics for Indexing') metron_service.check_kafka_topics(self.__params, self.__get_topics()) http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 1629465..9f9ab87 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -48,6 +48,7 @@ class Indexing(Script): env.set_params(params) Logger.info("Running indexing configure") + metron_service.check_indexer_parameters() File(format("{metron_config_path}/elasticsearch.properties"), content=Template("elasticsearch.properties.j2"), owner=params.metron_user, @@ -137,6 +138,7 @@ class Indexing(Script): def restart(self, env): from params import params env.set_params(params) + self.configure(env) commands = IndexingCommands(params) commands.restart_indexing_topology(env) @@ -145,6 +147,7 @@ class Indexing(Script): from params import params env.set_params(params) Logger.info("Installing Elasticsearch index templates") + metron_service.check_indexer_parameters() commands = IndexingCommands(params) for template_name, template_path in commands.get_templates().iteritems(): @@ -160,9 +163,11 @@ class Indexing(Script): from params import params env.set_params(params) Logger.info("Deleting Elasticsearch index templates") + metron_service.check_indexer_parameters() commands = IndexingCommands(params) for template_name in commands.get_templates(): + # delete the index template cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""; Execute( @@ -173,9 +178,9 @@ class Indexing(Script): def kibana_dashboard_install(self, env): from params import params env.set_params(params) + metron_service.check_indexer_parameters() Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url)) - kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template') if not os.path.isfile(kibanaTemplate): raise IOError( @@ -200,8 +205,9 @@ class Indexing(Script): def zeppelin_notebook_import(self, env): from params import params env.set_params(params) - commands = IndexingCommands(params) + metron_service.check_indexer_parameters() + commands = IndexingCommands(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index 23482e3..4a55f8a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -27,6 +27,7 @@ from resource_management.core.resources.system import Execute from resource_management.core.source import InlineTemplate from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.functions.get_user_call_output import get_user_call_output +from resource_management.libraries.script import Script from metron_security import kinit @@ -51,10 +52,23 @@ def set_configured(user, flag_file, log_msg): def set_zk_configured(params): set_configured(params.metron_user, params.zk_configured_flag_file, "Setting Zookeeper configured to true") -def build_global_config_patch(params, patch_file): - # see RFC 6902 at https://tools.ietf.org/html/rfc6902 - patch_template = """ - [ +def solr_global_config_patches(): + """ + Builds the global configuration patches required for Solr. + """ + return """ + { + "op": "add", + "path": "/solr.zookeeper", + "value": "{{solr_zookeeper_url}}" + } + """ + +def elasticsearch_global_config_patches(): + """ + Builds the global configuration patches required for Elasticsearch. + """ + return """ { "op": "add", "path": "/es.clustername", @@ -69,11 +83,31 @@ def build_global_config_patch(params, patch_file): "op": "add", "path": "/es.date.format", "value": "{{es_date_format}}" + } + """ + +def build_global_config_patch(params, patch_file): + """ + Build the file used to patch the global configuration. + See RFC 6902 at https://tools.ietf.org/html/rfc6902 + + :param params: + :param patch_file: The path where the patch file will be created. + """ + if params.ra_indexing_writer == 'Solr': + indexing_patches = solr_global_config_patches() + else: + indexing_patches = elasticsearch_global_config_patches() + other_patches = """ + { + "op": "add", + "path": "/profiler.client.period.duration", + "value": "{{profiler_period_duration}}" }, { "op": "add", - "path": "/solr.zookeeper", - "value": "{{solr_zookeeper_url}}" + "path": "/profiler.client.period.duration.units", + "value": "{{profiler_period_units}}" }, { "op": "add", @@ -92,16 +126,6 @@ def build_global_config_patch(params, patch_file): }, { "op": "add", - "path": "/profiler.client.period.duration", - "value": "{{profiler_period_duration}}" - }, - { - "op": "add", - "path": "/profiler.client.period.duration.units", - "value": "{{profiler_period_units}}" - }, - { - "op": "add", "path": "/user.settings.hbase.table", "value": "{{user_settings_hbase_table}}" }, @@ -115,8 +139,14 @@ def build_global_config_patch(params, patch_file): "path": "/bootstrap.servers", "value": "{{kafka_brokers}}" } - ] """ + patch_template = ambari_format( + """ + [ + {indexing_patches}, + {other_patches} + ] + """) File(patch_file, content=InlineTemplate(patch_template), owner=params.metron_user, @@ -132,6 +162,7 @@ def patch_global_config(params): "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PATCH --config_type GLOBAL --patch_file " + patch_file), path=ambari_format("{java_home}/bin") ) + Logger.info("Done patching global config") def pull_config(params): Logger.info('Pulling all Metron configs down from ZooKeeper to local file system') @@ -141,17 +172,12 @@ def pull_config(params): path=ambari_format("{java_home}/bin") ) -# pushes json patches to zookeeper based on Ambari parameters that are configurable by the user def refresh_configs(params): if not is_zk_configured(params): Logger.warning("The expected flag file '" + params.zk_configured_flag_file + "'indicating that Zookeeper has been configured does not exist. Skipping patching. An administrator should look into this.") return - - Logger.info("Patch global config in Zookeeper") + check_indexer_parameters() patch_global_config(params) - Logger.info("Done patching global config") - - Logger.info("Pull zookeeper config locally") pull_config(params) def get_running_topologies(params): @@ -505,3 +531,32 @@ def check_http(host, port, user): Execute(cmd, tries=3, try_sleep=5, logoutput=False, user=user) except: raise ComponentIsNotRunning() + +def check_indexer_parameters(): + """ + Ensure that all required parameters have been defined for the chosen + Indexer; either Solr or Elasticsearch. + """ + missing = [] + config = Script.get_config() + indexer = config['configurations']['metron-indexing-env']['ra_indexing_writer'] + Logger.info('Checking parameters for indexer = ' + indexer) + + if indexer == 'Solr': + # check for all required solr parameters + if not config['configurations']['metron-env']['solr_zookeeper_url']: + missing.append("metron-env/solr_zookeeper_url") + + else: + # check for all required elasticsearch parameters + if not config['configurations']['metron-env']['es_cluster_name']: + missing.append("metron-env/es_cluster_name") + if not config['configurations']['metron-env']['es_hosts']: + missing.append("metron-env/es_hosts") + if not config['configurations']['metron-env']['es_binary_port']: + missing.append("metron-env/es_binary_port") + if not config['configurations']['metron-env']['es_date_format']: + missing.append("metron-env/es_date_format") + + if len(missing) > 0: + raise Fail("Missing required indexing parameters(s): indexer={0}, missing={1}".format(indexer, missing))
