METRON-1423 Ambari work to handle Solr configuration (merrimanr) closes apache/metron#934
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/84cac7e8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/84cac7e8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/84cac7e8 Branch: refs/heads/master Commit: 84cac7e85f6b8421063f329d647be9bc82ea5f4c Parents: 1006d9d Author: merrimanr <[email protected]> Authored: Wed Feb 28 16:37:30 2018 -0600 Committer: merrimanr <[email protected]> Committed: Wed Feb 28 16:37:30 2018 -0600 ---------------------------------------------------------------------- .../ansible/playbooks/metron_full_install.yml | 4 ++ .../ansible/playbooks/solr_install.yml | 40 ++++++++++++ .../roles/ambari_config/vars/single_node_vm.yml | 1 + .../ansible/roles/solr/defaults/main.yml | 23 +++++++ .../ansible/roles/solr/meta/main.yml | 19 ++++++ .../ansible/roles/solr/tasks/main.yml | 36 +++++++++++ metron-deployment/packaging/ambari/.gitignore | 1 + .../packaging/ambari/metron-mpack/pom.xml | 8 ++- .../METRON/CURRENT/configuration/metron-env.xml | 6 ++ .../configuration/metron-indexing-env.xml | 32 +++++++--- .../common-services/METRON/CURRENT/metainfo.xml | 4 ++ .../package/scripts/indexing_commands.py | 25 +++++++- .../CURRENT/package/scripts/indexing_master.py | 67 ++++++++++++++++---- .../CURRENT/package/scripts/metron_service.py | 5 ++ .../package/scripts/params/params_linux.py | 15 ++++- .../package/scripts/params/status_params.py | 3 + .../METRON/CURRENT/themes/metron_theme.json | 18 ++++-- .../docker/rpm-docker/SPECS/metron.spec | 4 ++ .../src/main/config/elasticsearch.properties.j2 | 2 +- metron-platform/metron-solr/README.md | 19 ++---- .../src/main/config/solr.properties.j2 | 2 +- .../src/main/scripts/create_collection.sh | 27 ++++++++ .../src/main/scripts/delete_collection.sh | 27 ++++++++ .../src/main/scripts/install_solr.sh | 2 +- .../metron-solr/src/main/scripts/start_solr.sh | 27 ++++++++ .../metron-solr/src/main/scripts/stop_solr.sh | 27 ++++++++ pom.xml | 2 +- 27 files changed, 401 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/playbooks/metron_full_install.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml b/metron-deployment/ansible/playbooks/metron_full_install.yml index b517671..92a6473 100644 --- a/metron-deployment/ansible/playbooks/metron_full_install.yml +++ b/metron-deployment/ansible/playbooks/metron_full_install.yml @@ -25,6 +25,10 @@ tags: - build +- include: solr_install.yml + tags: + - solr + - include: ambari_install.yml tags: - ambari http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/playbooks/solr_install.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/playbooks/solr_install.yml b/metron-deployment/ansible/playbooks/solr_install.yml new file mode 100644 index 0000000..464ec80 --- /dev/null +++ b/metron-deployment/ansible/playbooks/solr_install.yml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- hosts: ec2 + become: true + tasks: + - include_vars: ../../amazon-ec2/conf/defaults.yml + tags: + - ec2 + +- hosts: packer + become: true + tasks: + - include_vars: ../../development/centos6/ansible/inventory/group_vars/all + tags: + - packer + +# +# Solr +# +- hosts: search + become: true + roles: + - { role: solr, tags: ['solr'] } + tags: + - solr http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml index bf54fe0..9addc5e 100644 --- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml @@ -116,6 +116,7 @@ required_configurations: storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744" es_hosts: "{{ groups.search | join(',') }}" zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995" + solr_zookeeper_url: "{{ groups.search[0] }}:9983" - metron-rest-env: metron_jdbc_driver: "org.h2.Driver" metron_jdbc_url: "jdbc:h2:file:~/metrondb" http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/solr/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/solr/defaults/main.yml b/metron-deployment/ansible/roles/solr/defaults/main.yml new file mode 100644 index 0000000..61562f8 --- /dev/null +++ b/metron-deployment/ansible/roles/solr/defaults/main.yml @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +solr_version: 6.6.2 +solr_installation_user: "solr" +solr_user_home: /var/solr +solr_home: "{{ solr_user_home }}/solr-{{ solr_version }}" +solr_url: http://archive.apache.org/dist/lucene/solr/{{ solr_version }}/solr-{{ solr_version }}.tgz + http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/solr/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/solr/meta/main.yml b/metron-deployment/ansible/roles/solr/meta/main.yml new file mode 100644 index 0000000..ddf6aa9 --- /dev/null +++ b/metron-deployment/ansible/roles/solr/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - java_jdk http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/solr/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/solr/tasks/main.yml b/metron-deployment/ansible/roles/solr/tasks/main.yml new file mode 100644 index 0000000..cf47177 --- /dev/null +++ b/metron-deployment/ansible/roles/solr/tasks/main.yml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Create Solr user + user: + name: solr + home: "{{ solr_user_home }}" + system: yes + +- name: Download Solr + get_url: + url: "{{ solr_url }}" + dest: "{{ solr_user_home }}/solr-{{ solr_version }}.tar.gz" + +- name: Extract Solr tarball + unarchive: + src: "{{ solr_user_home }}/solr-{{ solr_version }}.tar.gz" + dest: "{{ solr_user_home }}" + copy: no + creates: "{{ solr_home }}" + owner: "{{ solr_installation_user }}" + group: "{{ solr_installation_user }}" http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/.gitignore ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore index ca2e75c..72538a0 100644 --- a/metron-deployment/packaging/ambari/.gitignore +++ b/metron-deployment/packaging/ambari/.gitignore @@ -1,5 +1,6 @@ archive.zip *.hash elasticsearch.properties.j2 +solr.properties.j2 hdfs.properties.j2 enrichment.properties.j2 http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml index 491e8dd..e32063b 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml @@ -128,7 +128,13 @@ </includes> <filtering>false</filtering> </resource> - + <resource> + <directory>${basedir}/../../../../metron-platform/metron-solr/src/main/config</directory> + <includes> + <include>solr.properties.j2</include> + </includes> + <filtering>false</filtering> + </resource> </resources> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 87866e8..8a52583 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -100,6 +100,12 @@ <description>Elasticsearch HTTP port. (9200)</description> <display-name>Elasticsearch HTTP port</display-name> </property> + <property> + <name>solr_zookeeper_url</name> + <value>{{zookeeper_quorum}}</value> + <description>Comma delimited list of Zookeeper Urls: zkhost1:2181,zkhost1:2181</description> + <display-name>Solr Zookeeper Urls</display-name> + </property> <property require-input = "true"> <name>storm_rest_addr</name> <display-name>Storm Rest Server Address</display-name> http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml index b960536..3d268b7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -22,7 +22,7 @@ <name>ra_indexing_kafka_start</name> <description>Indexing Topology Spout Offset</description> <value>UNCOMMITTED_EARLIEST</value> - <display-name>Elasticsearch Indexing Offset</display-name> + <display-name>Random Access Indexing Offset</display-name> <value-attributes> <type>value-list</type> <entries> @@ -80,10 +80,22 @@ <display-name>Indexing Error Topic</display-name> </property> <property> - <name>ra_indexing_writer_class_name</name> - <description>Indexing Writer Class Name</description> - <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value> - <display-name>Indexing Writer Class Name</display-name> + <name>ra_indexing_writer</name> + <description>Search Engine Used for Random Access</description> + <value>Elasticsearch</value> + <display-name>Random Access Search Engine</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>Elasticsearch</value> + </entry> + <entry> + <value>Solr</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> </property> <property> <name>update_hbase_table</name> @@ -101,7 +113,7 @@ <name>ra_indexing_workers</name> <description>Number of Indexing Topology Workers</description> <value>1</value> - <display-name>Indexing Workers for Elasticsearch</display-name> + <display-name>Indexing Workers for Random Access</display-name> </property> <property> <name>batch_indexing_workers</name> @@ -113,7 +125,7 @@ <name>ra_indexing_acker_executors</name> <description>Number of Indexing Topology Ackers</description> <value>1</value> - <display-name>Enrichment Ackers for Elasticsearch</display-name> + <display-name>Enrichment Ackers for Random Access</display-name> </property> <property> <name>batch_indexing_acker_executors</name> @@ -135,7 +147,7 @@ <name>ra_indexing_topology_max_spout_pending</name> <description>Indexing Topology Spout Max Pending Tuples</description> <value/> - <display-name>Indexing Max Pending for Elasticsearch</display-name> + <display-name>Indexing Max Pending for Random Access</display-name> <value-attributes> <empty-value-valid>true</empty-value-valid> </value-attributes> @@ -152,7 +164,7 @@ <property> <name>ra_indexing_kafka_spout_parallelism</name> - <description>Indexing Topology Kafka Spout Parallelism for Elasticsearch</description> + <description>Indexing Topology Kafka Spout Parallelism for Random Access</description> <value>1</value> <display-name>Indexing Spout Parallelism</display-name> </property> @@ -164,7 +176,7 @@ </property> <property> <name>ra_indexing_writer_parallelism</name> - <description>Indexing Topology Writer Bolt Parallelism for Elasticsearch</description> + <description>Indexing Topology Writer Bolt Parallelism for Random Access</description> <value>1</value> <display-name>Indexing Writer Parallelism</display-name> </property> http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index 2b5b405..ff2d7d6 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -267,6 +267,7 @@ <scriptType>PYTHON</scriptType> </commandScript> <configuration-dependencies> + <config-type>metron-indexing-env</config-type> <config-type>metron-rest-env</config-type> </configuration-dependencies> </component> @@ -396,6 +397,9 @@ <name>metron-elasticsearch</name> </package> <package> + <name>metron-solr</name> + </package> + <package> <name>metron-pcap</name> </package> <package> http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 4c862f0..7af0f51 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -54,6 +54,7 @@ class IndexingCommands: self.__hbase_configured = os.path.isfile(self.__params.indexing_hbase_configured_flag_file) self.__hbase_acl_configured = os.path.isfile(self.__params.indexing_hbase_acl_configured_flag_file) self.__elasticsearch_template_installed = os.path.isfile(self.__params.elasticsearch_template_installed_flag_file) + self.__solr_schema_installed = os.path.isfile(self.__params.solr_schema_installed_flag_file) self.__hdfs_perm_configured = os.path.isfile(self.__params.indexing_hdfs_perm_configured_flag_file) def __get_topics(self): @@ -78,6 +79,21 @@ class IndexingCommands: "metaalert_index": params.meta_index_path } + def get_solr_schemas(self): + """ + Defines the Solr schemas. + :return: Dict where key is the name of a collection and the + value is a path to file containing the schema definition. + """ + from params import params + return { + "bro": params.bro_schema_path, + "yaf": params.yaf_schema_path, + "snort": params.snort_schema_path, + "error": params.error_schema_path, + "metaalert": params.meta_schema_path + } + def is_configured(self): return self.__configured @@ -96,6 +112,9 @@ class IndexingCommands: def is_elasticsearch_template_installed(self): return self.__elasticsearch_template_installed + def is_solr_schema_installed(self): + return self.__solr_schema_installed + def set_configured(self): metron_service.set_configured(self.__params.metron_user, self.__params.indexing_configured_flag_file, "Setting Indexing configured to True") @@ -114,6 +133,9 @@ class IndexingCommands: def set_elasticsearch_template_installed(self): metron_service.set_configured(self.__params.metron_user, self.__params.elasticsearch_template_installed_flag_file, "Setting Elasticsearch template installed to True") + def set_solr_schema_installed(self): + metron_service.set_configured(self.__params.metron_user, self.__params.solr_schema_installed_flag_file, "Setting Solr schema installed to True") + def create_hbase_tables(self): Logger.info("Creating HBase Tables for indexing") metron_service.create_hbase_table(self.__params, @@ -206,8 +228,9 @@ class IndexingCommands: self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) - start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh""" + if self.__params.ra_indexing_writer == 'Solr': + start_cmd_template = """{0}/bin/start_solr_topology.sh""" start_cmd = start_cmd_template.format(self.__params.metron_home) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index ce8c074..e63ea2d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -49,6 +49,11 @@ class Indexing(Script): owner=params.metron_user, group=params.metron_group ) + File(format("{metron_config_path}/solr.properties"), + content=Template("solr.properties.j2"), + owner=params.metron_user, + group=params.metron_group + ) File(format("{metron_config_path}/hdfs.properties"), content=Template("hdfs.properties.j2"), owner=params.metron_user, @@ -87,17 +92,29 @@ class Indexing(Script): env.set_params(params) self.configure(env) commands = IndexingCommands(params) - - # Install elasticsearch templates - try: - if not commands.is_elasticsearch_template_installed(): - self.elasticsearch_template_install(env) - commands.set_elasticsearch_template_installed() - - except Exception as e: - msg = "WARNING: Elasticsearch index templates could not be installed. " \ - "Is Elasticsearch running? Will reattempt install on next start. error={0}" - Logger.warning(msg.format(e)) + if params.ra_indexing_writer == 'Solr': + Logger.info("Loading Solr schemas") + # Install Solr schemas + try: + if not commands.is_solr_schema_installed(): + self.solr_schema_install(env) + commands.set_solr_schema_installed() + + except Exception as e: + msg = "WARNING: Solr schemas could not be installed. " \ + "Is Solr running? Will reattempt install on next start. error={0}" + Logger.warning(msg.format(e)) + else: + # Install elasticsearch templates + try: + if not commands.is_elasticsearch_template_installed(): + self.elasticsearch_template_install(env) + commands.set_elasticsearch_template_installed() + + except Exception as e: + msg = "WARNING: Elasticsearch index templates could not be installed. " \ + "Is Elasticsearch running? Will reattempt install on next start. error={0}" + Logger.warning(msg.format(e)) commands.start_indexing_topology(env) @@ -149,6 +166,34 @@ class Indexing(Script): cmd.format(params.es_http_url, template_name), logoutput=True) + def solr_schema_install(self, env): + from params import params + env.set_params(params) + Logger.info("Installing Solr schemas") + + commands = IndexingCommands(params) + for collection_name, config_path in commands.get_solr_schemas().iteritems(): + + # install the schema + + cmd = "{0}/bin/solr create -c {1} -d {2}" + Execute( + cmd.format(params.solr_home, collection_name, config_path), + logoutput=True, user="solr") + + def solr_schema_delete(self, env): + from params import params + env.set_params(params) + Logger.info("Deleting Solr schemas") + + commands = IndexingCommands(params) + for collection_name, config_path in commands.get_solr_schemas().iteritems(): + # delete the schema + cmd = "{0}/bin/solr delete -c {1}" + Execute( + cmd.format(params.solr_home, collection_name), + logoutput=True, user="solr") + def zeppelin_notebook_import(self, env): from params import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index 330d3c0..1f7ec6b 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -72,6 +72,11 @@ def build_global_config_patch(params, patch_file): }, { "op": "add", + "path": "/solr.zookeeper", + "value": "{{solr_zookeeper_url}}" + }, + { + "op": "add", "path": "/parser.error.topic", "value": "{{parser_error_topic}}" }, http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 3506fab..5bfa1dc 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -84,6 +84,7 @@ indexing_hbase_configured_flag_file = status_params.indexing_hbase_configured_fl indexing_hbase_acl_configured_flag_file = status_params.indexing_hbase_acl_configured_flag_file indexing_hdfs_perm_configured_flag_file = status_params.indexing_hdfs_perm_configured_flag_file elasticsearch_template_installed_flag_file = status_params.elasticsearch_template_installed_flag_file +solr_schema_installed_flag_file = status_params.solr_schema_installed_flag_file global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] # Elasticsearch hosts and port management @@ -120,6 +121,11 @@ if has_zk_host: # last port config zookeeper_quorum += ':' + zookeeper_clientPort +# Solr params +solr_version = '6.6.2' +solr_home = '/var/solr/solr-' + solr_version +solr_zookeeper_url = format(format(config['configurations']['metron-env']['solr_zookeeper_url'])) + # Storm storm_rest_addr = status_params.storm_rest_addr @@ -200,6 +206,13 @@ yaf_index_path = tmp_dir + "/yaf_index.template" error_index_path = tmp_dir + "/error_index.template" meta_index_path = tmp_dir + "/metaalert_index.template" +# Solr Schemas +bro_schema_path = metron_home + "/config/schema/bro" +snort_schema_path = metron_home + "/config/schema/snort" +yaf_schema_path = metron_home + "/config/schema/yaf" +error_schema_path = metron_home + "/config/schema/error" +meta_schema_path = metron_home + "/config/schema/metaalert" + # Zeppelin Notebooks metron_config_zeppelin_path = format("{metron_config_path}/zeppelin") zeppelin_shiro_ini_content = status_params.zeppelin_shiro_ini_content @@ -305,7 +318,7 @@ indexing_input_topic = status_params.indexing_input_topic indexing_error_topic = config['configurations']['metron-indexing-env']['indexing_error_topic'] metron_random_access_indexing_topology = status_params.metron_random_access_indexing_topology metron_batch_indexing_topology = status_params.metron_batch_indexing_topology -ra_indexing_writer_class_name = config['configurations']['metron-indexing-env']['ra_indexing_writer_class_name'] +ra_indexing_writer = config['configurations']['metron-indexing-env']['ra_indexing_writer'] batch_indexing_writer_class_name = config['configurations']['metron-indexing-env']['batch_indexing_writer_class_name'] ra_indexing_workers = config['configurations']['metron-indexing-env']['ra_indexing_workers'] batch_indexing_workers = config['configurations']['metron-indexing-env']['batch_indexing_workers'] http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 62cfc7a..1e5c35f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -79,6 +79,9 @@ indexing_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../me # Elasticsearch elasticsearch_template_installed_flag_file = metron_zookeeper_config_path + '/../metron_elasticsearch_template_installed_flag_file' +# Solr +solr_schema_installed_flag_file = metron_zookeeper_config_path + '/../metron_solr_schema_installed_flag_file' + # REST metron_rest_port = config['configurations']['metron-rest-env']['metron_rest_port'] http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index cef9a3b..ae15c31 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -183,7 +183,7 @@ "subsections": [ { "name": "subsection-indexing-storm", - "display-name": "Index Writer - Elasticsearch", + "display-name": "Index Writer - Random Access", "row-index": "0", "column-index": "0", "row-span": "1", @@ -391,6 +391,10 @@ "subsection-name": "subsection-index-settings" }, { + "config": "metron-env/solr_zookeeper_url", + "subsection-name": "subsection-index-settings" + }, + { "config": "metron-parsers-env/parsers", "subsection-name": "subsection-parsers" }, @@ -503,7 +507,7 @@ "subsection-name": "subsection-indexing-update" }, { - "config": "metron-indexing-env/ra_indexing_writer_class_name", + "config": "metron-indexing-env/ra_indexing_writer", "subsection-name": "subsection-indexing-storm" }, { @@ -711,6 +715,12 @@ } }, { + "config": "metron-env/solr_zookeeper_url", + "widget": { + "type": "text-field" + } + }, + { "config": "metron-parsers-env/parsers", "widget": { "type": "text-field" @@ -888,9 +898,9 @@ } }, { - "config": "metron-indexing-env/ra_indexing_writer_class_name", + "config": "metron-indexing-env/ra_indexing_writer", "widget": { - "type": "text-field" + "type": "combo" } }, { http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 5f97f62..5222efc 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -225,8 +225,12 @@ This package installs the Metron Solr files %dir %{metron_home}/bin %dir %{metron_home}/config %dir %{metron_home}/lib +%{metron_home}/bin/create_collection.sh +%{metron_home}/bin/delete_collection.sh %{metron_home}/bin/install_solr.sh +%{metron_home}/bin/start_solr.sh %{metron_home}/bin/start_solr_topology.sh +%{metron_home}/bin/stop_solr.sh %{metron_home}/config/solr.properties %{metron_home}/config/schema/bro/managed-schema %{metron_home}/config/schema/bro/solrconfig.xml http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2 b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2 index 00ad9dc..023ba2f 100644 --- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2 +++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2 @@ -35,7 +35,7 @@ indexing.input.topic={{indexing_input_topic}} indexing.error.topic={{indexing_error_topic}} ##### Indexing ##### -indexing.writer.class.name={{ra_indexing_writer_class_name}} +indexing.writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter ##### Parallelism ##### kafka.spout.parallelism={{ra_indexing_kafka_spout_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md index 164160e..4321114 100644 --- a/metron-platform/metron-solr/README.md +++ b/metron-platform/metron-solr/README.md @@ -72,10 +72,10 @@ via the global config. The following settings are possible as part of the globa ## Installing -A script is provided in the installation for installing Solr Cloud in quick-start mode in the [full dev environment for CentOS](../../metron-deployment/development/centos6). -The script is installed as part of the Solr RPM. Metron's Ambari MPack does not currently manage Solr installation in any way, so -you must first install the RPM in order to run the Solr setup script. +Solr is installed in the [full dev environment for CentOS](../../metron-deployment/development/centos6) by default but is not started initially. Navigate to `$METRON_HOME/bin` +and start Solr Cloud by running `start_solr.sh`. +Metron's Ambari MPack installs several scripts in `$METRON_HOME/bin` that can be used to manage Solr. A script is also provided for installing Solr Cloud outside of full dev. The script performs the following tasks * Stops ES and Kibana @@ -85,16 +85,9 @@ The script performs the following tasks _Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html_ -Login to the full dev environment as root and execute the following to install the Solr RPM. - -``` -rpm -ivh /localrepo/metron-solr-*.rpm -``` - -This will lay down the necessary files to setup Solr Cloud. Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`. - -After running this script, Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud. This manner -of starting Solr will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html) +Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`. After running this script, +Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud. This manner of starting Solr +will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html) ## Schemas http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/config/solr.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/solr.properties.j2 b/metron-platform/metron-solr/src/main/config/solr.properties.j2 index 00ad9dc..170aef3 100644 --- a/metron-platform/metron-solr/src/main/config/solr.properties.j2 +++ b/metron-platform/metron-solr/src/main/config/solr.properties.j2 @@ -35,7 +35,7 @@ indexing.input.topic={{indexing_input_topic}} indexing.error.topic={{indexing_error_topic}} ##### Indexing ##### -indexing.writer.class.name={{ra_indexing_writer_class_name}} +indexing.writer.class.name=org.apache.metron.solr.writer.SolrWriter ##### Parallelism ##### kafka.spout.parallelism={{ra_indexing_kafka_spout_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/create_collection.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/src/main/scripts/create_collection.sh new file mode 100755 index 0000000..37fbccb --- /dev/null +++ b/metron-platform/metron-solr/src/main/scripts/create_collection.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +METRON_VERSION=${project.version} +METRON_HOME=/usr/metron/$METRON_VERSION +SOLR_VERSION=${global_solr_version} +SOLR_USER=solr +SOLR_SERVICE=$SOLR_USER +SOLR_VAR_DIR="/var/$SOLR_SERVICE" + +cd $SOLR_VAR_DIR/solr-${SOLR_VERSION} +su $SOLR_USER -c "bin/solr create -c $1 -d $METRON_HOME/config/schema/$1/" http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/delete_collection.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh new file mode 100755 index 0000000..68f0c20 --- /dev/null +++ b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +METRON_VERSION=${project.version} +METRON_HOME=/usr/metron/$METRON_VERSION +SOLR_VERSION=${global_solr_version} +SOLR_USER=solr +SOLR_SERVICE=$SOLR_USER +SOLR_VAR_DIR="/var/$SOLR_SERVICE" + +cd $SOLR_VAR_DIR/solr-${SOLR_VERSION} +su $SOLR_USER -c "bin/solr delete -c $1" http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/install_solr.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/scripts/install_solr.sh b/metron-platform/metron-solr/src/main/scripts/install_solr.sh index 73415c8..da04557 100755 --- a/metron-platform/metron-solr/src/main/scripts/install_solr.sh +++ b/metron-platform/metron-solr/src/main/scripts/install_solr.sh @@ -30,7 +30,7 @@ service kibana stop service elasticsearch stop -SOLR_VERSION=6.6.2 +SOLR_VERSION=${global_solr_version} SOLR_USER=solr SOLR_SERVICE=$SOLR_USER SOLR_VAR_DIR="/var/$SOLR_SERVICE" http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/start_solr.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/scripts/start_solr.sh b/metron-platform/metron-solr/src/main/scripts/start_solr.sh new file mode 100755 index 0000000..6a89508 --- /dev/null +++ b/metron-platform/metron-solr/src/main/scripts/start_solr.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +METRON_VERSION=${project.version} +METRON_HOME=/usr/metron/$METRON_VERSION +SOLR_VERSION=${global_solr_version} +SOLR_USER=solr +SOLR_SERVICE=$SOLR_USER +SOLR_VAR_DIR="/var/$SOLR_SERVICE" + +cd $SOLR_VAR_DIR/solr-${SOLR_VERSION} +su $SOLR_USER -c "bin/solr -e cloud -noprompt" http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/stop_solr.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/scripts/stop_solr.sh b/metron-platform/metron-solr/src/main/scripts/stop_solr.sh new file mode 100755 index 0000000..5fe8d9c --- /dev/null +++ b/metron-platform/metron-solr/src/main/scripts/stop_solr.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +METRON_VERSION=${project.version} +METRON_HOME=/usr/metron/$METRON_VERSION +SOLR_VERSION=${global_solr_version} +SOLR_USER=solr +SOLR_SERVICE=$SOLR_USER +SOLR_VAR_DIR="/var/$SOLR_SERVICE" + +cd $SOLR_VAR_DIR/solr-${SOLR_VERSION} +su $SOLR_USER -c "bin/solr stop -all" http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3ee242c..38738b5 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ <global_slf4j_version>1.7.7</global_slf4j_version> <global_opencsv_version>3.7</global_opencsv_version> <global_java_version>1.8</global_java_version> - <global_solr_version>6.6.0</global_solr_version> + <global_solr_version>6.6.2</global_solr_version> <global_mockito_version>1.10.19</global_mockito_version> <global_shade_version>2.4.3</global_shade_version> <global_jackson_version>2.7.4</global_jackson_version>
