http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml deleted file mode 100644 index be9b1d3..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Copy Metron bundles - copy: - src: "{{ item }}" - dest: "{{ metron_directory }}" - with_items: - - "{{ metron_solr_bundle_path }}" - - "{{ metron_elasticsearch_bundle_path }}" - - "{{ metron_enrichment_bundle_path }}" - - "{{ metron_indexing_bundle_path }}" - - "{{ metron_parsers_bundle_path }}" - - "{{ metron_data_management_bundle_path }}" - - "{{ metron_common_bundle_path }}" - - "{{ metron_pcap_bundle_path }}" - - "{{ metron_maas_bundle_path }}" - - "{{ metron_profiler_bundle_path }}" - - "{{ metron_profiler_client_bundle_path }}" - -- name: Unbundle Metron bundles - shell: "{{ item }}" - args: - chdir: "{{ metron_directory }}" - with_items: - - "tar xzvf {{ metron_solr_bundle_name }}" - - "tar xzvf {{ metron_elasticsearch_bundle_name }}" - - "tar xzvf {{ metron_enrichment_bundle_name }}" - - "tar xzvf {{ metron_indexing_bundle_name }}" - - "tar xzvf {{ metron_parsers_bundle_name }}" - - "tar xzvf {{ metron_data_management_bundle_name }}" - - "tar xzvf {{ metron_common_bundle_name }}" - - "tar xzvf {{ metron_pcap_bundle_name }}" - - "tar xzvf {{ metron_maas_bundle_name }}" - - "tar xzvf {{ metron_profiler_bundle_name }}" - - "tar xvzf {{ metron_profiler_bundle_name }} " - - "tar xvzf {{ metron_profiler_client_bundle_name }} " - - rm *.tar.gz \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/es_purge.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/es_purge.yml b/metron-deployment/roles/metron_streaming/tasks/es_purge.yml deleted file mode 100644 index 22616ca..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/es_purge.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create Empty Log Files for ES Purge - file: - path: "{{ item }}" - state: touch - owner: hdfs - group: hdfs - mode: 0644 - with_items: - - /var/log/bro-purge/cron-es-bro-purge.log - - /var/log/yaf-purge/cron-es-yaf-purge.log - - /var/log/snort-purge/cron-es-snort-purge.log - - -- name: Purge Elasticsearch Indices every 30 days. - cron: - name: "{{ item.name }}" - job: "{{ item.job }}" - special_time: daily - user: hdfs - with_items: - - { name: "bro_es_purge", job: "{{ es_bro_purge_cronjob }}" } - - { name: "yaf_es_purge", job: "{{ es_yaf_purge_cronjob }}" } - - { name: "snort_es_purge", job: "{{ es_snort_purge_cronjob }}" } - - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/geoip.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/geoip.yml b/metron-deployment/roles/metron_streaming/tasks/geoip.yml deleted file mode 100644 index b26f889..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/geoip.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- - -- name: Load Geo DB - shell: "{{ metron_directory }}/bin/geo_enrichment_load.sh -z {{ zookeeper_url }}" - become: yes - become_user: hdfs http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml b/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml deleted file mode 100644 index d857bf5..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create HDFS directory for grok patterns - command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Assign hfds user as owner of {{ metron_hdfs_output_dir }}/patterns HDFS directory - command: hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Assign permissions of HDFS {{ metron_hdfs_output_dir }}/patterns directory - command: hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns - become: yes - become_user: hdfs - -- name: Upload Grok Patterns to hdfs://{{ metron_hdfs_output_dir }} - command: hdfs dfs -put -f {{ metron_directory }}/patterns {{ metron_hdfs_output_dir }} - become: yes - become_user: hdfs - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml b/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml deleted file mode 100644 index 125d41d..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml +++ /dev/null @@ -1,56 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create root user HDFS directory - command: hdfs dfs -mkdir -p /user/root - become: yes - become_user: hdfs - -- name: Assign root as owner of /user/root HDFS directory - command: hdfs dfs -chown root:root /user/root - become: yes - become_user: hdfs - -- name: Create Metron HDFS output directory - command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }} - become: yes - become_user: hdfs - -- name: Create Metron HDFS geo directory - command: hdfs dfs -mkdir -p {{ geo_hdfs_path }} - become: yes - become_user: hdfs - -- name: Assign hdfs as owner of HDFS output directory - command: hdfs dfs -chown hdfs:hadoop {{ metron_hdfs_output_dir }} - become: yes - become_user: hdfs - -- name: Assign hdfs as owner of HDFS Geo directory - command: hdfs dfs -chown hdfs:hadoop {{ geo_hdfs_path }} - become: yes - become_user: hdfs - -- name: Assign permissions of HDFS output directory - command: hdfs dfs -chmod 775 {{ metron_hdfs_output_dir }} - become: yes - become_user: hdfs - -- name: Assign permissions of HDFS geo directory - command: hdfs dfs -chmod 775 {{ geo_hdfs_path }} - become: yes - become_user: hdfs \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml b/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml deleted file mode 100644 index 33442e4..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create Log Directories for HDFS Purge - file: - path: "{{ item }}" - state: directory - mode: 0755 - owner: hdfs - group: hdfs - with_items: - - /var/log/bro-purge - - /var/log/yaf-purge - - /var/log/snort-purge - -- name: Create Empty Log Files for HDFS Purge - file: - path: "{{ item }}" - state: touch - owner: hdfs - group: hdfs - mode: 0644 - with_items: - - /var/log/bro-purge/cron-hdfs-bro-purge.log - - /var/log/yaf-purge/cron-hdfs-yaf-purge.log - - /var/log/snort-purge/cron-hdfs-snort-purge.log - -- name: Purge HDFS Sensor Data every 30 days. - cron: - name: "{{ item.name }}" - job: "{{ item.job }}" - special_time: daily - user: hdfs - with_items: - - { name: "bro_hdfs_purge", job: "{{ hdfs_bro_purge_cronjob }}" } - - { name: "yaf_hdfs_purge", job: "{{ hdfs_yaf_purge_cronjob }}" } - - { name: "snort_hdfs_purge", job: "{{ hdfs_snort_purge_cronjob }}" } - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml deleted file mode 100644 index 498b8dd..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/main.yml +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create Metron directories - file: path={{ metron_directory }}/{{ item }} state=directory mode=0755 - with_items: - - lib - - bin - - config - -- name: Create Source Config Directory - file: - path: "{{ zookeeper_config_path }}" - state: directory - mode: 0755 - -- include: copy_bundles.yml - -- include: hdfs_filesystem.yml - run_once: true - -- include: grok_upload.yml - run_once: true - -- include: topologies.yml - -- include: source_config.yml - run_once: true - -- include: geoip.yml - run_once: true - -- include: threat_intel.yml - run_once: true - when: threat_intel_bulk_load == True - -- include: hdfs_purge.yml - -- include: es_purge.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/source_config.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/source_config.yml b/metron-deployment/roles/metron_streaming/tasks/source_config.yml deleted file mode 100644 index 1c967bd..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/source_config.yml +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- - -- name: Copy Elasticsearch Global Config File - template: - src: "templates/config/elasticsearch.global.json" - dest: "{{ zookeeper_global_config_path }}" - mode: 0644 - when: install_elasticsearch | default(False) == True - -- name: Copy Solr Global Config File - template: - src: "../roles/metron_streaming/templates/config/solr.global.json" - dest: "{{ zookeeper_global_config_path }}" - mode: 0644 - when: install_solr | default(False) == True - -- name: Load Config - shell: "{{ metron_directory }}/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {{ zookeeper_url }} && touch {{ zookeeper_config_path }}/configured" - - - - - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml b/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml deleted file mode 100644 index a1aa237..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- - -- name: Create Bulk load working Directory - file: - path: "{{ threat_intel_work_dir }}" - state: directory - -- name: Copy extractor.json to {{ inventory_hostname }} - copy: - src: ../roles/metron_streaming/files/extractor.json - dest: "{{ threat_intel_work_dir }}" - mode: 0644 - -- name: Copy Bulk Load CSV File - template: - src: "{{ threat_intel_csv_filepath }}" - dest: "{{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }}" - mode: 0644 - -- name: Copy Bulk Load CSV File to HDFS - command: "hdfs dfs -put -f {{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }} ." - -- name: Run Threat Intel Bulk Load - shell: "{{ threat_intel_bin }} -c t -t {{threatintel_hbase_table}} -e {{ threat_intel_work_dir }}/extractor.json -i /user/root -m MR && touch {{ threat_intel_work_dir }}/loaded" - args: - creates: "{{ threat_intel_work_dir }}/loaded" - -- name: Clean up HDFS File - command: "hdfs dfs -rm {{ threat_intel_csv_filename }}" - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/topologies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/tasks/topologies.yml b/metron-deployment/roles/metron_streaming/tasks/topologies.yml deleted file mode 100644 index 827e861..0000000 --- a/metron-deployment/roles/metron_streaming/tasks/topologies.yml +++ /dev/null @@ -1,86 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Configure Metron Pcap Topology - lineinfile: > - dest={{ metron_pcap_properties_config_path }} - regexp="{{ item.regexp }}" - line="{{ item.line }}" - with_items: - - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } - - { regexp: "kafka.pcap.out=", line: "kafka.pcap.out={{ pcap_hdfs_path }}" } - - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" } - -- name: Configure Metron Enrichment topology - lineinfile: > - dest={{ metron_enrichment_properties_config_path }} - regexp="{{ item.regexp }}" - line="{{ item.line }}" - with_items: - - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } - - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" } - - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" } - - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" } - - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" } - - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" } - - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" } - - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" } - - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" } - - -- name: Configure Metron Solr topology - lineinfile: > - dest={{ metron_solr_properties_config_path }} - regexp="{{ item.regexp }}" - line="{{ item.line }}" - with_items: - - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } - - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" } - - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" } - - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/indexing/indexed" } - - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" } - - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" } - - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" } - -- name: Configure Metron Elasticsearch topology - lineinfile: > - dest={{ metron_elasticsearch_properties_config_path }} - regexp="{{ item.regexp }}" - line="{{ item.line }}" - with_items: - - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } - - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" } - - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" } - - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" } - - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" } - - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" } - - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/indexing/indexed" } - - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" } - - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" } - - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" } - -- name: Configure Profiler topology - lineinfile: > - dest={{ metron_profiler_properties_config_path }} - regexp="{{ item.regexp }}" - line="{{ item.line }}" - with_items: - - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } - - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" } - - - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json b/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json deleted file mode 100644 index 87af1c0..0000000 --- a/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "es.clustername": "{{ elasticsearch_cluster_name }}", - "es.ip": "{{ groups.search[0] }}", - "es.port": "{{ elasticsearch_transport_port }}", - "es.date.format": "yyyy.MM.dd.HH", - "parser.error.topic": "indexing" -} http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/templates/config/solr.global.json ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/templates/config/solr.global.json b/metron-deployment/roles/metron_streaming/templates/config/solr.global.json deleted file mode 100644 index 5cb7a4d..0000000 --- a/metron-deployment/roles/metron_streaming/templates/config/solr.global.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "solr.zookeeper": "{{ zookeeper_url }}", - "solr.collection": "{{ solr_collection_name }}", - "solr.numShards": {{ solr_number_shards }}, - "solr.replicationFactor": {{ solr_replication_factor }} -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/templates/threat_ip.csv ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/metron_streaming/templates/threat_ip.csv b/metron-deployment/roles/metron_streaming/templates/threat_ip.csv deleted file mode 100644 index 3ac38f3..0000000 --- a/metron-deployment/roles/metron_streaming/templates/threat_ip.csv +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#Add single column of ip address to alert -#Public lists are available on the internet -# example: -23.113.113.105 -24.107.205.249 -24.108.62.255 -24.224.153.71 -27.4.1.212 -27.131.149.102 -31.24.30.31 -31.131.251.33 -31.186.99.250 -31.192.209.119 -31.192.209.150 -31.200.244.17 -37.34.52.185 -37.58.112.101 -37.99.146.27 -37.128.132.96 -37.140.195.177 -37.140.199.100 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit-start/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit-start/defaults/main.yml b/metron-deployment/roles/monit-start/defaults/main.yml deleted file mode 100644 index 26a05b3..0000000 --- a/metron-deployment/roles/monit-start/defaults/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- - -# -# defines which services will be started. by default, no services -# are started -# -services_to_start: [] http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit-start/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit-start/tasks/main.yml b/metron-deployment/roles/monit-start/tasks/main.yml deleted file mode 100644 index 68bf07a..0000000 --- a/metron-deployment/roles/monit-start/tasks/main.yml +++ /dev/null @@ -1,53 +0,0 @@ - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Start monit - service: - name: monit - state: started - enabled: true - -- name: Reload monit definitions - shell: monit reload - -- name: Stop all services - shell: monit stop all - register: result - until: result.rc == 0 - retries: 20 - delay: 10 - -- name: Find the services installed on the host - shell: monit summary | tail -n +3 | awk -F"'" '{print $2}' - register: installed_services - -# which services that need started are actually installed on this host? -- set_fact: - installed_services_to_start: "{{ services_to_start | intersect(installed_services.stdout_lines) }}" - -- name: Start Metron services - debug: - msg: "Attemping to start: {{ installed_services_to_start }}" - -- name: Wait for metron services to start - shell: "monit start {{ item }}" - with_items: "{{ installed_services_to_start }}" - register: result - until: result.rc == 0 - retries: 20 - delay: 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/README.md b/metron-deployment/roles/monit/README.md deleted file mode 100644 index 6f50852..0000000 --- a/metron-deployment/roles/monit/README.md +++ /dev/null @@ -1,79 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> -# Monit Integration - -This role will leverage Monit as a process watchdog to manage sensors, topologies, and core services. - -* Monit can be used to start, stop, or check status of any of the sensors or topologies. -* When monitoring is enabled (on by default) if a process dies, it will be restarted. -* The Monit web interface is exposed at http://hostname:2812. -* The web interface username and password is defined by the `monit_user` and `monit_password` variables. These default to `admin`/`monit`. -* Monit CLI tools can also be used to simplify the process of managing Metron components. -* The post-deployment report for Amazon-EC2 provides links to Monit's web interface labeled as 'Sensor Status' and 'Topology Status.' - - ``` - ok: [localhost] => { - "Success": [ - "Apache Metron deployed successfully", - " Metron @ http://ec2-52-39-143-62.us-west-2.compute.amazonaws.com:5000";, - " Ambari @ http://ec2-52-39-4-93.us-west-2.compute.amazonaws.com:8080";, - " Sensor Status @ http://ec2-52-39-4-93.us-west-2.compute.amazonaws.com:2812";, - " Topology Status @ http://ec2-52-39-130-62.us-west-2.compute.amazonaws.com:2812";, - "For additional information, see https://metron.apache.org/'" - ] - } - ``` - -## Usage - - -Start all Metron components - -``` -monit start all -``` - -Stop all Metron components - -``` -monit stop all -``` - -Start an individual Metron component - -``` -monit start bro-parser -``` - -Start all components required to ingest Bro data - -``` -monit -g bro start -``` - -Start all parsers - -``` -monit -g parsers start -``` - -What is running? - -``` -monit summary -``` http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/defaults/main.yml b/metron-deployment/roles/monit/defaults/main.yml deleted file mode 100644 index 651aa58..0000000 --- a/metron-deployment/roles/monit/defaults/main.yml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -monit_home: /usr/local/monit -monit_config_home: /etc/monit.d -monit_user: admin -monit_pass: monit - -bro_pid_file: /usr/local/bro/spool/bro/.pid -snort_alert_csv_path: /var/log/snort/alert.csv http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/tasks/main.yml b/metron-deployment/roles/monit/tasks/main.yml deleted file mode 100644 index 9c1d75d..0000000 --- a/metron-deployment/roles/monit/tasks/main.yml +++ /dev/null @@ -1,28 +0,0 @@ - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- include: monit.yml -- include: monit-definitions.yml - -- include: monit-sensor-definitions.yml - tags: - - sensors - -- include: monit-stub-definitions.yml - tags: - - sensor-stubs http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit-definitions.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml deleted file mode 100644 index 13e00fc..0000000 --- a/metron-deployment/roles/monit/tasks/monit-definitions.yml +++ /dev/null @@ -1,22 +0,0 @@ - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create monit definition for pcap-replay - template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit - when: ("sensors" in group_names) and (install_pcap_replay | default(False)) - tags: sensors http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml b/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml deleted file mode 100644 index 4b2d5de..0000000 --- a/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml +++ /dev/null @@ -1,38 +0,0 @@ - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create monit definition for pcap-replay - template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit - when: ("sensors" in group_names) and (install_pcap_replay | default(False)) - -- name: Create monit definition for pycapa - template: src=monit/pycapa.monit dest={{ monit_config_home }}/pycapa.monit - when: ("sensors" in group_names) and (install_pycapa | default(True)) - -- name: Create monit definition for snort - template: src=monit/snort.monit dest={{ monit_config_home }}/snort.monit - when: ("sensors" in group_names) and (install_snort | default(True)) - -- name: Create monit definition for yaf - template: src=monit/yaf.monit dest={{ monit_config_home }}/yaf.monit - when: ("sensors" in group_names) and (install_yaf | default(True)) - -- name: Create monit definition for bro - template: src=monit/bro.monit dest={{ monit_config_home }}/bro.monit - when: ("sensors" in group_names) and (install_bro | default(True)) - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml b/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml deleted file mode 100644 index fde711a..0000000 --- a/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml +++ /dev/null @@ -1,30 +0,0 @@ - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create monit definition for snort - template: src=monit/snort-stub.monit dest={{ monit_config_home }}/snort.monit - when: ("sensors" in group_names) and (install_snort | default(True)) - -- name: Create monit definition for yaf - template: src=monit/yaf-stub.monit dest={{ monit_config_home }}/yaf.monit - when: ("sensors" in group_names) and (install_yaf | default(True)) - -- name: Create monit definition for bro - template: src=monit/bro-stub.monit dest={{ monit_config_home }}/bro.monit - when: ("sensors" in group_names) and (install_bro | default(True)) - http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/tasks/monit.yml b/metron-deployment/roles/monit/tasks/monit.yml deleted file mode 100644 index bf300f4..0000000 --- a/metron-deployment/roles/monit/tasks/monit.yml +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Install monit - yum: - name: monit - register: result - until: result.rc == 0 - retries: 5 - delay: 10 - -- name: Deploy monit configuration - template: src=monit/monit.conf dest=/etc/monit.conf http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/bro-stub.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/bro-stub.monit b/metron-deployment/roles/monit/templates/monit/bro-stub.monit deleted file mode 100644 index 54bdbd6..0000000 --- a/metron-deployment/roles/monit/templates/monit/bro-stub.monit +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -check process bro - with pidfile /var/run/sensor-stubs-bro.pid - start program = "/etc/init.d/sensor-stubs start bro" - stop program = "/etc/init.d/sensor-stubs stop bro" - if does not exist then restart - group bro - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/bro.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/bro.monit b/metron-deployment/roles/monit/templates/monit/bro.monit deleted file mode 100644 index db6a318..0000000 --- a/metron-deployment/roles/monit/templates/monit/bro.monit +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check process bro - with pidfile {{ bro_pid_file }} - start program = "/usr/local/bro/bin/broctl deploy" - stop program = "/usr/local/bro/bin/broctl stop" - restart program = "/usr/local/bro/bin/broctl restart" - if does not exist then restart - group bro - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/monit.conf ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/monit.conf b/metron-deployment/roles/monit/templates/monit/monit.conf deleted file mode 100644 index 660c421..0000000 --- a/metron-deployment/roles/monit/templates/monit/monit.conf +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set logfile syslog -set pidfile /var/run/monit.pid - -# check services at X second intervals -set daemon 30 -include /etc/monit.d/* - -# allow http access -set httpd port 2812 and - use address {{ inventory_hostname }} - allow 0.0.0.0/0 - allow {{ monit_user }}:{{ monit_pass }} http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/pcap-replay.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/pcap-replay.monit b/metron-deployment/roles/monit/templates/monit/pcap-replay.monit deleted file mode 100644 index da0c006..0000000 --- a/metron-deployment/roles/monit/templates/monit/pcap-replay.monit +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check process pcap-replay - with pidfile /var/run/pcap-replay.pid - start program = "/etc/init.d/pcap-replay start" - stop program = "/etc/init.d/pcap-replay stop" - if does not exist then restart - group pcap - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/pycapa.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/pycapa.monit b/metron-deployment/roles/monit/templates/monit/pycapa.monit deleted file mode 100644 index 1123d87..0000000 --- a/metron-deployment/roles/monit/templates/monit/pycapa.monit +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check process pycapa - with pidfile /var/run/pycapa.pid - start program = "/etc/init.d/pycapa start" - stop program = "/etc/init.d/pycapa stop" - if does not exist then restart - group pcap - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/snort-stub.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/snort-stub.monit b/metron-deployment/roles/monit/templates/monit/snort-stub.monit deleted file mode 100644 index b782690..0000000 --- a/metron-deployment/roles/monit/templates/monit/snort-stub.monit +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -check process snort - with pidfile /var/run/sensor-stubs-snort.pid - start program = "/etc/init.d/sensor-stubs start snort" - stop program = "/etc/init.d/sensor-stubs stop snort" - if does not exist then restart - group snort - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/snort.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/snort.monit b/metron-deployment/roles/monit/templates/monit/snort.monit deleted file mode 100644 index 6fb429b..0000000 --- a/metron-deployment/roles/monit/templates/monit/snort.monit +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check process snort matching "/usr/sbin/snort" - start program = "/etc/init.d/snortd start" - stop program = "/etc/init.d/snortd stop" - if does not exist then restart - group snort - group sensors - group metron - -check process snort-producer - with pidfile /var/run/snort-producer.pid - start program = "/etc/init.d/snort-producer start" - stop program = "/etc/init.d/snort-producer stop" - if does not exist then restart - depends on snort - group snort - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/yaf-stub.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/yaf-stub.monit b/metron-deployment/roles/monit/templates/monit/yaf-stub.monit deleted file mode 100644 index 2a92a53..0000000 --- a/metron-deployment/roles/monit/templates/monit/yaf-stub.monit +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -check process yaf - with pidfile /var/run/sensor-stubs-yaf.pid - start program = "/etc/init.d/sensor-stubs start yaf" - stop program = "/etc/init.d/sensor-stubs stop yaf" - if does not exist then restart - group yaf - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/yaf.monit ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/monit/templates/monit/yaf.monit b/metron-deployment/roles/monit/templates/monit/yaf.monit deleted file mode 100644 index 1f7b4d2..0000000 --- a/metron-deployment/roles/monit/templates/monit/yaf.monit +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -check process yaf - with pidfile /var/run/yaf.pid - start program = "/etc/init.d/yaf start" - stop program = "/etc/init.d/yaf stop" - if does not exist then restart - group yaf - group sensors - group metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ntp/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ntp/tasks/main.yml b/metron-deployment/roles/ntp/tasks/main.yml deleted file mode 100644 index 7b1b9a8..0000000 --- a/metron-deployment/roles/ntp/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Install ntp - yum: - name: ntp - state: present - register: result - until: result.rc == 0 - retries: 5 - delay: 10 - -- name: Ensure ntp is running and enabled - service: - name: ntpd - state: started - enabled: yes http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/README.md b/metron-deployment/roles/opentaxii/README.md deleted file mode 100644 index 7c111de..0000000 --- a/metron-deployment/roles/opentaxii/README.md +++ /dev/null @@ -1,178 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> -# OpenTAXII - -Installs [OpenTAXII](https://github.com/EclecticIQ/OpenTAXII) as a deamon that can be launched via a SysV service script. The complementary client implementation, [Cabby](https://github.com/EclecticIQ/cabby) is also installed. - -OpenTAXII is a robust Python implementation of TAXII Services that delivers a rich feature set and friendly pythonic API. [TAXII](https://stixproject.github.io/) (Trusted Automated eXchange of Indicator Information) is a collection of specifications defining a set of services and message exchanges used for sharing cyber threat intelligence information between parties. - -## Getting Started - -After deployment completes the OpenTAXII service is installed and running. A set of [Hail a TAXII](http://hailataxii.com/) threat intel collections have been defined and configured. Use the `status` option to view the collections that have been defined. - -``` -$ service opentaxii status -Checking opentaxii... Running -guest.phishtank_com 0 -guest.Abuse_ch 0 -guest.CyberCrime_Tracker 0 -guest.EmergingThreats_rules 0 -guest.Lehigh_edu 0 -guest.MalwareDomainList_Hostlist 0 -guest.blutmagie_de_torExits 0 -guest.dataForLast_7daysOnly 0 -guest.dshield_BlockList 0 -``` - -Notice that each collections contain zero records. None of the data is automatically synced during deployment. To sync the data manually use the `sync` option as defined below. The following example does not provide a begin and end time so the data will be fetched for the current day only. - -``` -# service opentaxii sync guest.blutmagie_de_torExits -2016-04-21 20:34:42,511 INFO: Starting new HTTP connection (1): localhost -2016-04-21 20:34:42,540 INFO: Response received for Inbox_Message from http://localhost:9000/services/inbox -2016-04-21 20:34:42,542 INFO: Sending Inbox_Message to http://localhost:9000/services/inbox -... -2016-04-21 20:34:42,719 INFO: Response received for Poll_Request from http://localhost:9000/services/poll -2016-04-21 20:34:42,719 INFO: Content blocks count: 1618, is partial: False -``` - -The OpenTAXII service now contains 1,618 threat intel records indicating Tor Exit nodes. - -``` -[root@source ~]# service opentaxii status -Checking opentaxii... Running -guest.phishtank_com 0 -guest.Abuse_ch 0 -guest.CyberCrime_Tracker 0 -guest.EmergingThreats_rules 0 -guest.Lehigh_edu 0 -guest.MalwareDomainList_Hostlist 0 -guest.blutmagie_de_torExits 1618 -guest.dataForLast_7daysOnly 0 -guest.dshield_BlockList 0 -``` - -## Usage - -A standard SysV script has been installed to manage OpenTAXII. The following functions are available. - -`start` `stop` `restart` the OpenTAXII service - -`status` of the OpenTAXII service. The command displays the collections that have been defined and the number of records in each. - -``` -$ service opentaxii status -Checking opentaxii... Running -guest.phishtank_com 984 -guest.Abuse_ch 45 -guest.CyberCrime_Tracker 482 -guest.EmergingThreats_rules 0 -guest.Lehigh_edu 1030 -guest.MalwareDomainList_Hostlist 84 -guest.blutmagie_de_torExits 3236 -guest.dataForLast_7daysOnly 3377 -guest.dshield_BlockList 0 -``` - -`setup` Initializes the services and collections required to operate the OpenTAXII service. This will destroy all existing data. The user is prompted to continue before any data is destroyed. - -``` -# service opentaxii setup -WARNING: force reset and destroy all opentaxii data? [Ny]: y -Stopping opentaxii ..Ok -2016-04-21T19:56:01.886157Z [opentaxii.server] info: api.persistence.loaded {timestamp=2016-04-21T19:56:01.886157Z, logger=opentaxii.server, api_class=SQLDatabaseAPI, event=api.persistence.loaded, level=info} -2016-04-21T19:56:01.896503Z [opentaxii.server] info: api.auth.loaded {timestamp=2016-04-21T19:56:01.896503Z, logger=opentaxii.server, api_class=SQLDatabaseAPI, event=api.auth.loaded, level=info} -2016-04-21T19:56:01.896655Z [opentaxii.server] info: taxiiserver.configured {timestamp=2016-04-21T19:56:01.896655Z, logger=opentaxii.server, event=taxiiserver.configured, level=info} -... -Ok -``` - -`sync [collection] [begin-at] [end-at]` Syncs the threat intel data available at [Hail a TAXII](http://hailataxii.com/). If no begin and end date is provided then data is synced over the current day only. - - `collection` Name of the collection to sync. - - `begin-at` Exclusive begin of time window; ISO8601 - - `end-at` Inclusive end of time window; ISO8601 - -``` -$ service opentaxii sync guest.phishtank_com -+ /usr/local/opentaxii/opentaxii-venv/bin/taxii-proxy --poll-path http://hailataxii.com/taxii-data --poll-collection guest.phishtank_com --inbox-path http://localhost:9000/services/guest.phishtank_com-inbox --inbox-collection guest.phishtank_com --binding urn:stix.mitre.org:xml:1.1.1 --begin 2016-04-21 --end 2016-04-22 -2016-04-21 17:36:23,778 INFO: Sending Poll_Request to http://hailataxii.com/taxii-data -2016-04-21 17:36:23,784 INFO: Starting new HTTP connection (1): hailataxii.com -2016-04-21 17:36:24,175 INFO: Response received for Poll_Request from http://hailataxii.com/taxii-data -2016-04-21 17:36:24,274 INFO: Sending Inbox_Message to http://localhost:9000/services/guest.phishtank_com-inbox -... -2016-04-21 17:36:34,867 INFO: Response received for Poll_Request from http://localhost:9000/services/guest.phishtank_com-poll -2016-04-21 17:36:34,868 INFO: Content blocks count: 6993, is partial: False -``` - -### Troubleshooting - -Should you need to explore the installation, here are instructions on doing so. - -OpenTAXII is installed in a virtual environment. Before exploring the environment run the following commands to perform the necessary setup. The specific paths may change depending on your Ansible settings. - -``` -export LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64 -export OPENTAXII_CONFIG=/usr/local/opentaxii/etc/opentaxii-conf.yml -cd /usr/local/opentaxii -. opentaxii-venv/bin/activate -``` - -Discover available services. - -``` -taxii-discovery --discovery http://localhost:9000/services/discovery -taxii-discovery --discovery http://hailataxii.com/taxii-data -``` - -Explore available collections. - -``` -taxii-collections --discovery http://localhost:9000/services/discovery -taxii-collections --discovery http://hailataxii.com/taxii-data -``` - -Read data from a collection. - -``` -taxii-poll --discovery http://localhost:9000/services/discovery -c guest.phishtank_com -taxii-poll --discovery http://hailataxii.com/taxii-data -c guest.phishtank_com --begin 2016-04-20 -``` - -Manually load data into a collection. - -``` -taxii-push \ - --discovery http://localhost:9000/services/discovery \ - --dest phishtank \ - --content-file data.xml \ - --username guest \ - --password guest -``` - -Fetch data from a remote service and mirror it locally. - -``` -taxii-proxy --poll-path http://hailataxii.com/taxii-data \ - --poll-collection guest.phishtank_com \ - --inbox-path http://localhost:9000/services/guest.phishtank_com-inbox \ - --inbox-collection guest.phishtank_com \ - --binding urn:stix.mitre.org:xml:1.1.1 \ - --inbox-username guest \ - --inbox-password guest \ - --begin 2016-04-20 -``` http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/defaults/main.yml b/metron-deployment/roles/opentaxii/defaults/main.yml deleted file mode 100644 index 9ab86cb..0000000 --- a/metron-deployment/roles/opentaxii/defaults/main.yml +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -opentaxii_host: localhost -opentaxii_port: 9000 -opentaxii_domain: "http://{{ opentaxii_host }}:{{ opentaxii_port }}" -opentaxii_bind: "{{ opentaxii_host }}:{{ opentaxii_port }}" -opentaxii_home: /usr/local/opentaxii -opentaxii_venv: opentaxii-venv -opentaxii_bin: "{{ opentaxii_home }}/{{ opentaxii_venv }}/bin" -opentaxii_user: guest -opentaxii_pass: guest -opentaxii_workers: 2 -opentaxii_loglevel: info -opentaxii_timeout: 300 -opentaxii_auth_db: "{{ opentaxii_home }}/data/auth.db" -opentaxii_data_db: "{{ opentaxii_home }}/data/data.db" -opentaxii_salt: "@#L:KJDASLKJASD@" -python27_home: /opt/rh/python27/root -opentaxii_available_collections: - - guest.phishtank_com - - guest.Abuse_ch - - guest.CyberCrime_Tracker - - guest.EmergingThreats_rules - - guest.Lehigh_edu - - guest.MalwareDomainList_Hostlist - - guest.blutmagie_de_torExits - - guest.dataForLast_7daysOnly - - guest.dshield_BlockList http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/meta/main.yml b/metron-deployment/roles/opentaxii/meta/main.yml deleted file mode 100644 index 841d185..0000000 --- a/metron-deployment/roles/opentaxii/meta/main.yml +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/dependencies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/tasks/dependencies.yml b/metron-deployment/roles/opentaxii/tasks/dependencies.yml deleted file mode 100644 index 3b2b38a..0000000 --- a/metron-deployment/roles/opentaxii/tasks/dependencies.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Install yum repositories - yum: name={{ item }} update_cache=yes - with_items: - - epel-release - - centos-release-scl - -- name: Install dependencies - yum: name={{ item }} - with_items: - - "@Development tools" - - python27 - - python27-scldevel - - python27-python-virtualenv - - libxml2-devel - - libxslt-devel - - libselinux-python - register: result - until: result.rc == 0 - retries: 5 - delay: 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/hailataxii.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/tasks/hailataxii.yml b/metron-deployment/roles/opentaxii/tasks/hailataxii.yml deleted file mode 100644 index 1eebfe6..0000000 --- a/metron-deployment/roles/opentaxii/tasks/hailataxii.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Deploy service/collection definitions for hailataxii.com - template: src={{ item }} dest={{ opentaxii_home }}/etc mode=0400 - with_items: - - services.yml - - collections.yml - -- name: Add collection definitions for hailataxii.com - blockinfile: - dest: "{{ opentaxii_home }}/etc/collections.yml" - marker: "# {mark} ANSIBLE MANAGED BLOCK {{ item }}" - block: | - - name: {{ item }} - type: DATA_SET - available: true - accept_all_content: true - supported_content: - - urn:stix.mitre.org:xml:1.1.1 - service_ids: - - inbox - - collection - - poll - with_items: "{{ opentaxii_available_collections }}" - -- name: Setup opentaxii - shell: /etc/init.d/opentaxii setup - -- name: Start opentaxii - service: name=opentaxii state=restarted http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/tasks/main.yml b/metron-deployment/roles/opentaxii/tasks/main.yml deleted file mode 100644 index baa6b35..0000000 --- a/metron-deployment/roles/opentaxii/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- include: dependencies.yml -- include: opentaxii.yml -- include: hailataxii.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/opentaxii.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/tasks/opentaxii.yml b/metron-deployment/roles/opentaxii/tasks/opentaxii.yml deleted file mode 100644 index c153149..0000000 --- a/metron-deployment/roles/opentaxii/tasks/opentaxii.yml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Create install directory - file: path={{ item }} state=directory mode=0755 - with_items: - - "{{ opentaxii_home }}" - - "{{ opentaxii_home }}/etc" - - "{{ opentaxii_home }}/data" - - "{{ opentaxii_home }}/bin" - -- name: Create virtual environment - shell: "{{ python27_home }}/usr/bin/virtualenv {{ opentaxii_venv }}" - args: - chdir: "{{ opentaxii_home }}" - creates: "{{ opentaxii_home }}/{{ opentaxii_venv }}" - environment: - LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64" - -- name: Install opentaxii, cabby, and gunicorn - shell: "{{ opentaxii_bin }}/pip install {{ item }}" - environment: - LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64" - with_items: - - opentaxii - - cabby - - gunicorn - -- name: Deploy collection status script - template: src=collection-status.py dest={{ opentaxii_home }}/bin mode=0755 - -- name: Deploy opentaxii configs - template: src=opentaxii-conf.yml dest={{ opentaxii_home }}/etc mode=0400 - -- name: Deploy opentaxii service script - template: src=opentaxii dest=/etc/init.d/opentaxii mode=0755 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/collection-status.py ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/templates/collection-status.py b/metron-deployment/roles/opentaxii/templates/collection-status.py deleted file mode 100644 index 2d912c9..0000000 --- a/metron-deployment/roles/opentaxii/templates/collection-status.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from cabby import create_client - -try: - # create a connection - client = create_client(host='{{ opentaxii_host }}', port='{{ opentaxii_port }}', discovery_path='/services/discovery') - - # iterate through each defined collection - collections = client.get_collections(uri='{{ opentaxii_domain }}/services/collection') - - for collection in collections: - # how many records in each collection? - count = client.get_content_count(collection_name=collection.name, uri='{{ opentaxii_domain }}/services/poll') - print "%-50s %-10d" % (collection.name, count.count) -except: - print "Services not defined" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/collections.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/templates/collections.yml b/metron-deployment/roles/opentaxii/templates/collections.yml deleted file mode 100644 index 07a0e5a..0000000 --- a/metron-deployment/roles/opentaxii/templates/collections.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -collections: -# intentionally blank - managed by ansible http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/opentaxii ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/templates/opentaxii b/metron-deployment/roles/opentaxii/templates/opentaxii deleted file mode 100644 index e934e7f..0000000 --- a/metron-deployment/roles/opentaxii/templates/opentaxii +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# opentaxii daemon -# chkconfig: 345 20 80 -# description: OpenTAXII is a robust Python implementation of TAXII Service -# processname: opentaxii -# -NAME=opentaxii -DESC="OpenTAXII is a robust Python implementation of a TAXII service" -PIDFILE=/var/run/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -LOGFILE="/var/log/$NAME.log" -EXTRA_ARGS="${@:2}" -CONFIRM_TIMEOUT=3 -DAEMON_PATH="{{ opentaxii_home }}" - -export LD_LIBRARY_PATH={{ python27_home }}/usr/lib64 -export OPENTAXII_CONFIG={{ opentaxii_home }}/etc/opentaxii-conf.yml - -case "$1" in - - ############################################################################## - # start - # - start) - printf "%-50s" "Starting $NAME..." - - # setup virtual environment - cd $DAEMON_PATH - . {{ opentaxii_bin }}/activate - - # kick-off the daemon - DAEMON="{{ opentaxii_bin }}/gunicorn" - DAEMONOPTS="opentaxii.http:app" - DAEMONOPTS+=" --daemon" - DAEMONOPTS+=" --pid $PIDFILE" - DAEMONOPTS+=" --workers {{ opentaxii_workers }}" - DAEMONOPTS+=" --log-level {{ opentaxii_loglevel }}" - DAEMONOPTS+=" --log-file $LOGFILE" - DAEMONOPTS+=" --timeout {{ opentaxii_timeout }}" - DAEMONOPTS+=" --bind {{ opentaxii_bind }}" - DAEMONOPTS+=" --env OPENTAXII_CONFIG={{ opentaxii_home }}/etc/opentaxii-conf.yml" - DAEMONOPTS+=" $EXTRA_ARGS" - PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1` - printf "%s\n" "Ok" - ;; - - ############################################################################## - # status - # - status) - printf "%-50s" "Checking $NAME..." - . {{ opentaxii_bin }}/activate - if [ -f $PIDFILE ]; then - PID=`cat $PIDFILE` - if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then - printf "%s\n" "Process dead but pidfile exists" - else - printf "%s\n" "Running" - {{ opentaxii_home }}/bin/collection-status.py - fi - else - printf "%s\n" "Service not running" - fi - ;; - - ############################################################################## - # stop - # - stop) - printf "%-50s" "Stopping $NAME" - cd $DAEMON_PATH - if [ -f $PIDFILE ]; then - PID=`cat $PIDFILE` - while sleep 1 - echo -n "." - kill -0 $PID >/dev/null 2>&1 - do - kill $PID - done - - printf "%s\n" "Ok" - rm -f $PIDFILE - else - printf "%s\n" "pidfile not found" - fi - ;; - - ############################################################################## - # restart - # - restart) - $0 stop - $0 start - ;; - - ############################################################################## - # setup - # - setup) - - # if the database file already exists; prompt for confirmation - if [ -f "{{ opentaxii_data_db }}" ]; then - read -t $CONFIRM_TIMEOUT -p "WARNING: force reset and destroy all opentaxii data? [Ny]: " REPLY - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 0 - fi - fi - - $0 stop - cd $DAEMON_PATH - rm -f {{ opentaxii_auth_db }} - rm -f {{ opentaxii_data_db }} - {{ opentaxii_bin }}/opentaxii-create-account --username {{ opentaxii_user }} --password {{ opentaxii_pass }} - {{ opentaxii_bin }}/opentaxii-create-services -c {{ opentaxii_home}}/etc/services.yml - {{ opentaxii_bin }}/opentaxii-create-collections -c {{ opentaxii_home}}/etc/collections.yml - printf "%s\n" "Ok" - ;; - - ############################################################################## - # sync - # - sync) - - # collect the arguments - POLL_SOURCE="http://hailataxii.com/taxii-data"; - COLL="$2" - BEGIN="${3:-`date --iso-8601`}" - END="${4:-`date --date=tomorrow --iso-8601`}" - - # validation - if [ -z "$COLL" ]; then - echo "$0 sync [COLLECTION] [BEGIN-AT] [END-AT]" - echo "error: missing name of collection" - exit 1 - fi - - # sync the data - set -x - {{ opentaxii_bin }}/taxii-proxy \ - --poll-path $POLL_SOURCE \ - --poll-collection $COLL \ - --inbox-path {{ opentaxii_domain }}/services/inbox \ - --inbox-collection $COLL \ - --binding urn:stix.mitre.org:xml:1.1.1 \ - --begin $BEGIN \ - --end $END - set +x - - # count the number of records in the local collection - {{ opentaxii_bin }}/taxii-poll \ - --discovery {{ opentaxii_domain }}/services/discovery \ - --collection $COLL \ - --count-only - ;; - - *) - echo "Usage: $0 {status|start|stop|restart|setup|sync}" - exit 1 -esac
