Repository: metron Updated Branches: refs/remotes/apache/feature/METRON-1699-create-batch-profiler 113ac0d3a -> f5eaef3c2
METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e78bc530 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e78bc530 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e78bc530 Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: e78bc530260aa622e8263cda2b94be99194dffb5 Parents: 8763fba Author: merrimanr <merrim...@gmail.com> Authored: Mon Jul 23 09:08:12 2018 -0500 Committer: rmerriman <merrim...@gmail.com> Committed: Mon Jul 23 09:08:12 2018 -0500 ---------------------------------------------------------------------- .../CURRENT/configuration/metron-rest-env.xml | 25 ++++++++++++++++++++ .../package/scripts/params/params_linux.py | 5 +++- .../CURRENT/package/scripts/rest_commands.py | 18 ++++++++++++-- .../METRON/CURRENT/package/templates/metron.j2 | 4 ++++ .../METRON/CURRENT/themes/metron_theme.json | 10 ++++++++ .../src/main/config/rest_application.yml | 6 ++++- .../src/main/config/pcap.properties | 2 +- 7 files changed, 65 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 066b527..20f9767 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -168,4 +168,29 @@ <description>The field name where the threat triage score can be found in the search indices. This setting primarily affects the Alerts UI.</description> <value>threat:triage:score</value> </property> + <property> + <name>pcap_base_path</name> + <display-name>Pcap Base Path</display-name> + <description>The HDFS path where pcap files are stored. These files are the input to pcap queries.</description> + <value>/apps/metron/pcap/input</value> + </property> + <property> + <name>pcap_base_interim_result_path</name> + <display-name>Pcap Base Interim Result Path</display-name> + <description>The HDFS path where interim results of a pcap query are stored, before they are separated into pages.</description> + <value>/apps/metron/pcap/interim</value> + </property> + <property> + <name>pcap_final_output_path</name> + <display-name>Pcap Final Output Path</display-name> + <description>The HDFS path where the final results of a pcap query are stored.</description> + <value>/apps/metron/pcap/output</value> + </property> + <property> + <name>pcap_page_size</name> + <display-name>Pcap Page Size</display-name> + <description>The number of pcaps written to a page/file as a result of a pcap query.</description> + <value>10</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index b8d6123..73d3469 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -386,7 +386,10 @@ bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env' bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count'] # Pcap -pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap") +pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path'] +pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path'] +pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path'] +pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] pcap_configured_flag_file = status_params.pcap_configured_flag_file # MapReduce http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 2fd3f0d..463dca1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -117,8 +117,22 @@ class RestCommands: metron_service.init_kafka_acl_groups(self.__params, groups) def init_pcap(self): - Logger.info("Creating HDFS location for Pcap") - self.__params.HdfsResource(self.__params.pcap_hdfs_dir, + Logger.info("Creating HDFS locations for Pcap") + self.__params.HdfsResource(self.__params.pcap_base_path, + type="directory", + action="create_on_execute", + owner=self.__params.metron_user, + group=self.__params.metron_group, + mode=0755, + ) + self.__params.HdfsResource(self.__params.pcap_base_interim_result_path, + type="directory", + action="create_on_execute", + owner=self.__params.metron_user, + group=self.__params.metron_group, + mode=0755, + ) + self.__params.HdfsResource(self.__params.pcap_final_output_path, type="directory", action="create_on_execute", owner=self.__params.metron_user, http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 049b6ee..278d6f8 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -40,3 +40,7 @@ SECURITY_ENABLED={{security_enabled|lower}} KAFKA_SECURITY_PROTOCOL="{{kafka_security_protocol}}" PARSER_TOPOLOGY_OPTIONS="/home/{{metron_user}}/.storm/storm.config" METRON_ESCALATION_TOPIC="{{metron_escalation_topic}}" +PCAP_BASE_PATH="{{pcap_base_path}}" +PCAP_BASE_INTERIM_RESULT_PATH="{{pcap_base_interim_result_path}}" +PCAP_FINAL_OUTPUT_PATH="{{pcap_final_output_path}}" +PCAP_PAGE_SIZE="{{pcap_page_size}}" http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 17b4460..9f5b04e 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -820,6 +820,10 @@ "subsection-name": "subsection-rest" }, { + "config": "metron-rest-env/pcap_page_size", + "subsection-name": "subsection-rest" + }, + { "config": "metron-management-ui-env/metron_management_ui_port", "subsection-name": "subsection-management-ui" }, @@ -1421,6 +1425,12 @@ } }, { + "config": "metron-rest-env/pcap_page_size", + "widget": { + "type": "text-field" + } + }, + { "config": "metron-management-ui-env/metron_management_ui_port", "widget": { "type": "text-field" http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-interface/metron-rest/src/main/config/rest_application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml index 3999393..7486112 100644 --- a/metron-interface/metron-rest/src/main/config/rest_application.yml +++ b/metron-interface/metron-rest/src/main/config/rest_application.yml @@ -55,4 +55,8 @@ kerberos: keytab: ${METRON_SERVICE_KEYTAB} pcap: - pdml.script.path: ${METRON_HOME}/bin/pcap_to_pdml.sh \ No newline at end of file + pdml.script.path: ${METRON_HOME}/bin/pcap_to_pdml.sh + base.path: ${PCAP_BASE_PATH} + base.interim.result.path: ${PCAP_BASE_INTERIM_RESULT_PATH} + final.output.path: ${PCAP_FINAL_OUTPUT_PATH} + page.size: ${PCAP_PAGE_SIZE} http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-platform/metron-pcap-backend/src/main/config/pcap.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties index 7160178..848b588 100644 --- a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties +++ b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties @@ -27,6 +27,6 @@ kafka.pcap.start=UNCOMMITTED_EARLIEST kafka.pcap.numPackets=1000 kafka.pcap.maxTimeMS=300000 kafka.pcap.ts_scheme=FROM_KEY -kafka.pcap.out=/apps/metron/pcap +kafka.pcap.out=/apps/metron/pcap/input kafka.pcap.ts_granularity=MICROSECONDS kafka.spout.parallelism=1