Repository: metron Updated Branches: refs/heads/master fd896fbeb -> 2d5209ebf
METRON-1311 Service Check Should Check Elasticsearch Index Templates (nickwallen) closes apache/metron#839 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2d5209eb Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2d5209eb Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2d5209eb Branch: refs/heads/master Commit: 2d5209ebf7b7876a7da8a3908cb7808f0ad22615 Parents: fd896fb Author: nickwallen <[email protected]> Authored: Thu Nov 16 08:42:04 2017 -0500 Committer: nickallen <[email protected]> Committed: Thu Nov 16 08:42:04 2017 -0500 ---------------------------------------------------------------------- .../CURRENT/package/files/meta_index.template | 47 -------------- .../package/files/metaalert_index.template | 47 ++++++++++++++ .../package/scripts/indexing_commands.py | 29 +++++++++ .../CURRENT/package/scripts/indexing_master.py | 68 +++++--------------- .../package/scripts/params/params_linux.py | 2 +- 5 files changed, 94 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template deleted file mode 100644 index 964a480..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.template +++ /dev/null @@ -1,47 +0,0 @@ -{ - "template": "metaalert_index*", - "mappings": { - "metaalert_doc": { - "_timestamp": { - "enabled": true - }, - "dynamic_templates": [ - { - "alert_template": { - "path_match": "alert.*", - "match_mapping_type": "string", - "mapping": { - "type": "string", - "index": "not_analyzed" - } - } - } - ], - "properties": { - "guid": { - "type": "string", - "index": "not_analyzed" - }, - "score": { - "type": "string", - "index": "not_analyzed" - }, - "status": { - "type": "string", - "index": "not_analyzed" - }, - "timestamp": { - "type": "date", - "format": "epoch_millis" - }, - "alert": { - "type": "nested" - }, - "source:type": { - "type": "string", - "index": "not_analyzed" - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template new file mode 100644 index 0000000..964a480 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template @@ -0,0 +1,47 @@ +{ + "template": "metaalert_index*", + "mappings": { + "metaalert_doc": { + "_timestamp": { + "enabled": true + }, + "dynamic_templates": [ + { + "alert_template": { + "path_match": "alert.*", + "match_mapping_type": "string", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + } + ], + "properties": { + "guid": { + "type": "string", + "index": "not_analyzed" + }, + "score": { + "type": "string", + "index": "not_analyzed" + }, + "status": { + "type": "string", + "index": "not_analyzed" + }, + "timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "alert": { + "type": "nested" + }, + "source:type": { + "type": "string", + "index": "not_analyzed" + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index e6cfabb..5a2b0f4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -58,6 +58,21 @@ class IndexingCommands: # Indexed topic names matches the group return [self.__indexing_topic] + def get_templates(self): + """ + Defines the Elasticsearch index templates. + :return: Dict where key is the name of an index template and the + value is a path to file containing the index template definition. + """ + from params import params + return { + "bro_index": params.bro_index_path, + "yaf_index": params.yaf_index_path, + "snort_index": params.snort_index_path, + "error_index": params.error_index_path, + "metaalert_index": params.meta_index_path + } + def is_configured(self): return self.__configured @@ -159,6 +174,17 @@ class IndexingCommands: ) Logger.info('Done creating HDFS indexing directory') + def check_elasticsearch_templates(self): + for template_name in self.get_templates(): + + # check for the index template + cmd = "curl -s -XGET \"http://{0}/_template/{1}\"; | grep -o {1}" + err_msg="Missing Elasticsearch index template: name={0}" + metron_service.execute( + cmd=cmd.format(self.__params.es_http_url, template_name), + user=self.__params.metron_user, + err_msg=err_msg.format(template_name)) + def start_indexing_topology(self, env): Logger.info('Starting ' + self.__indexing_topology) @@ -241,6 +267,9 @@ class IndexingCommands: metron_service.check_hbase_table(self.__params, self.__params.update_hbase_table) metron_service.check_hbase_column_family(self.__params, self.__params.update_hbase_table, self.__params.update_hbase_cf) + Logger.info('Checking Elasticsearch templates for Indexing') + self.check_elasticsearch_templates() + if self.__params.security_enabled: Logger.info('Checking Kafka ACLs for Indexing') http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 92077ac..8992950 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -118,65 +118,31 @@ class Indexing(Script): def elasticsearch_template_install(self, env): from params import params env.set_params(params) + Logger.info("Installing Elasticsearch index templates") - File(params.bro_index_path, - mode=0755, - content=StaticFile('bro_index.template') - ) - - File(params.snort_index_path, - mode=0755, - content=StaticFile('snort_index.template') - ) - - File(params.yaf_index_path, - mode=0755, - content=StaticFile('yaf_index.template') - ) - - File(params.error_index_path, - mode=0755, - content=StaticFile('error_index.template') - ) - - File(params.meta_index_path, - mode=0755, - content=StaticFile('meta_index.template') - ) - - bro_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}') - Execute(bro_cmd, logoutput=True) - - snort_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}') - Execute(snort_cmd, logoutput=True) - - yaf_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}') - Execute(yaf_cmd, logoutput=True) - - error_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/error_index -d @{error_index_path}') - Execute(error_cmd, logoutput=True) + commands = IndexingCommands(params) + for template_name, template_path in commands.get_templates().iteritems(): - meta_cmd = ambari_format('curl -s -XPOST http://{es_http_url}/_template/metaalert_index -d @{meta_index_path}') - Execute(meta_cmd, logoutput=True) + # install the index template + File(template_path, mode=0755, content=StaticFile("{0}.template".format(template_name))) + cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}" + Execute( + cmd.format(params.es_http_url, template_name, template_path), + logoutput=True) def elasticsearch_template_delete(self, env): from params import params env.set_params(params) + Logger.info("Deleting Elasticsearch index templates") - bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/bro_index";') - Execute(bro_cmd, logoutput=True) - - snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/snort_index";') - Execute(snort_cmd, logoutput=True) - - yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/yaf_index";') - Execute(yaf_cmd, logoutput=True) - - error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/error_index";') - Execute(error_cmd, logoutput=True) + commands = IndexingCommands(params) + for template_name in commands.get_templates(): - meta_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/_template/metaalert_index";') - Execute(meta_cmd, logoutput=True) + # delete the index template + cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""; + Execute( + cmd.format(params.es_http_url, template_name), + logoutput=True) def zeppelin_notebook_import(self, env): from params import params http://git-wip-us.apache.org/repos/asf/metron/blob/2d5209eb/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 077a9c1..32d8889 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -195,7 +195,7 @@ bro_index_path = tmp_dir + "/bro_index.template" snort_index_path = tmp_dir + "/snort_index.template" yaf_index_path = tmp_dir + "/yaf_index.template" error_index_path = tmp_dir + "/error_index.template" -meta_index_path = tmp_dir + "/meta_index.template" +meta_index_path = tmp_dir + "/metaalert_index.template" # Zeppelin Notebooks metron_config_zeppelin_path = format("{metron_config_path}/zeppelin")
