Repository: metron Updated Branches: refs/heads/feature/METRON-1554-pcap-query-panel 77570464b -> 165c0e716
METRON-1555 Update REST to run YARN and MR jobs (merrimanr) closes apache/metron#1019 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/165c0e71 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/165c0e71 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/165c0e71 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 165c0e716342d743460123a9de6303178ed32830 Parents: 7757046 Author: merrimanr <merrim...@gmail.com> Authored: Tue Jun 26 10:18:01 2018 -0500 Committer: rmerriman <merrim...@gmail.com> Committed: Tue Jun 26 10:18:01 2018 -0500 ---------------------------------------------------------------------- dependencies_with_url.csv | 1 + .../package/scripts/params/params_linux.py | 10 ++ .../package/scripts/params/status_params.py | 6 + .../CURRENT/package/scripts/rest_commands.py | 37 ++++ .../CURRENT/package/scripts/rest_master.py | 4 + .../apache/metron/rest/model/PcapResponse.java | 38 +++++ .../rest/model/pcap/FixedPcapRequest.java | 115 +++++++++++++ .../metron/rest/model/pcap/PcapRequest.java | 91 ++++++++++ metron-interface/metron-rest/README.md | 8 + metron-interface/metron-rest/pom.xml | 33 ++++ .../apache/metron/rest/MetronRestConstants.java | 3 + .../apache/metron/rest/config/PcapConfig.java | 35 ++++ .../metron/rest/controller/PcapController.java | 53 ++++++ .../apache/metron/rest/service/PcapService.java | 27 +++ .../rest/service/impl/PcapServiceImpl.java | 120 +++++++++++++ .../src/main/resources/application.yml | 6 +- .../metron-rest/src/main/scripts/metron-rest.sh | 2 +- .../apache/metron/rest/config/TestConfig.java | 7 + .../PcapControllerIntegrationTest.java | 129 ++++++++++++++ .../apache/metron/rest/mock/MockPcapJob.java | 97 +++++++++++ .../rest/service/impl/PcapServiceImplTest.java | 167 +++++++++++++++++++ 21 files changed, 987 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/dependencies_with_url.csv ---------------------------------------------------------------------- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index df3bcd2..9fe3b2a 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -206,6 +206,7 @@ io.confluent:kafka-avro-serializer:jar:1.0:compile,ASLv2,https://github.com/conf io.confluent:kafka-schema-registry-client:jar:1.0:compile,ASLv2,https://github.com/confluentinc/schema-registry/ io.netty:netty-all:jar:4.0.23.Final:compile,ASLv2, io.netty:netty-all:jar:4.0.23.Final:provided,ASLv2, +io.netty:netty-all:jar:4.1.23.Final:compile,ASLv2, io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/ io.netty:netty:jar:3.6.2.Final:compile,Apache License, Version 2.0,http://netty.io/ io.netty:netty:jar:3.7.0.Final:compile,Apache License, Version 2.0,http://netty.io/ http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index f44d05f..a2d8185 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -39,6 +39,8 @@ import status_params config = Script.get_config() tmp_dir = Script.get_tmp_dir() +hdp_version = default("/commandParams/version", None) + hostname = config['hostname'] metron_home = status_params.metron_home @@ -357,3 +359,11 @@ metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-in bolt_hdfs_rotation_policy = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy'] bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_units'] bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count'] + +# Pcap +pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap") +pcap_configured_flag_file = status_params.pcap_configured_flag_file + +# MapReduce +metron_user_hdfs_dir = '/user/' + metron_user +metron_user_hdfs_dir_configured_flag_file = status_params.metron_user_hdfs_dir_configured_flag_file http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index ed2edfb..cb26a87 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -114,3 +114,9 @@ metron_user = config['configurations']['metron-env']['metron_user'] metron_principal_name = config['configurations']['metron-env']['metron_principal_name'] metron_keytab_path = config['configurations']['metron-env']['metron_service_keytab'] + +# Pcap +pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured' + +# MapReduce +metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index e97af05..73bafbb 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -37,6 +37,8 @@ class RestCommands: __kafka_acl_configured = False __hbase_configured = False __hbase_acl_configured = False + __pcap_configured = False + __metron_user_hdfs_dir_configured = False def __init__(self, params): if params is None: @@ -46,6 +48,8 @@ class RestCommands: self.__kafka_acl_configured = os.path.isfile(self.__params.rest_kafka_acl_configured_flag_file) self.__hbase_configured = os.path.isfile(self.__params.rest_hbase_configured_flag_file) self.__hbase_acl_configured = os.path.isfile(self.__params.rest_hbase_acl_configured_flag_file) + self.__pcap_configured = os.path.isfile(self.__params.pcap_configured_flag_file) + self.__metron_user_hdfs_dir_configured = os.path.isfile(self.__params.metron_user_hdfs_dir_configured_flag_file) Directory(params.metron_rest_pid_dir, mode=0755, owner=params.metron_user, @@ -74,6 +78,12 @@ class RestCommands: def is_hbase_acl_configured(self): return self.__hbase_acl_configured + def is_pcap_configured(self): + return self.__pcap_configured + + def is_metron_user_hdfs_dir_configured(self): + return self.__metron_user_hdfs_dir_configured + def set_kafka_configured(self): metron_service.set_configured(self.__params.metron_user, self.__params.rest_kafka_configured_flag_file, "Setting Kafka configured to True for rest") @@ -86,6 +96,12 @@ class RestCommands: def set_hbase_acl_configured(self): metron_service.set_configured(self.__params.metron_user, self.__params.rest_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for rest") + def set_pcap_configured(self): + metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting Pcap configured to True") + + def set_metron_user_hdfs_dir_configured(self): + metron_service.set_configured(self.__params.metron_user, self.__params.metron_user_hdfs_dir_configured_flag_file, "Setting Metron user HDFS directory configured to True") + def init_kafka_topics(self): Logger.info('Creating Kafka topics for rest') metron_service.init_kafka_topics(self.__params, self.__get_topics()) @@ -100,6 +116,26 @@ class RestCommands: groups = ['metron-rest'] metron_service.init_kafka_acl_groups(self.__params, groups) + def init_pcap(self): + Logger.info("Creating HDFS location for Pcap") + self.__params.HdfsResource(self.__params.pcap_hdfs_dir, + type="directory", + action="create_on_execute", + owner=self.__params.metron_user, + group=self.__params.metron_group, + mode=0755, + ) + + def create_metron_user_hdfs_dir(self): + Logger.info("Creating HDFS location for Metron user") + self.__params.HdfsResource(self.__params.metron_user_hdfs_dir, + type="directory", + action="create_on_execute", + owner=self.__params.metron_user, + group=self.__params.metron_group, + mode=0755, + ) + def start_rest_application(self): """ Start the REST application @@ -125,6 +161,7 @@ class RestCommands: "export METRON_INDEX_CP={metron_indexing_classpath};" "export METRON_LOG_DIR={metron_log_dir};" "export METRON_PID_FILE={pid_file};" + "export HDP_VERSION={hdp_version};" "{metron_home}/bin/metron-rest.sh;" "unset METRON_JDBC_PASSWORD;" )) http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py index 2f2d3f9..c842214 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py @@ -51,6 +51,10 @@ class RestMaster(Script): commands.init_kafka_topics() if not commands.is_hbase_configured(): commands.create_hbase_tables() + if not commands.is_pcap_configured(): + commands.init_pcap() + if not commands.is_metron_user_hdfs_dir_configured(): + commands.create_metron_user_hdfs_dir() if params.security_enabled and not commands.is_hbase_acl_configured(): commands.set_hbase_acls() if params.security_enabled and not commands.is_kafka_acl_configured(): http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java new file mode 100644 index 0000000..1d33675 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import java.util.ArrayList; +import java.util.List; + +public class PcapResponse { + + private List<byte[]> pcaps = new ArrayList<>(); + + public List<byte[]> getPcaps(){ + if(pcaps == null) { + return new ArrayList<>(); + } else { + return pcaps; + } + } + + public void setPcaps(List<byte[]> pcaps) { + this.pcaps = pcaps; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java new file mode 100644 index 0000000..758340b --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model.pcap; + +public class FixedPcapRequest extends PcapRequest { + + private String ipSrcAddr; + private String ipDstAddr; + private Integer ipSrcPort; + private Integer ipDstPort; + private String protocol; + private String packetFilter; + private Boolean includeReverse = false; + + public String getIpSrcAddr() { + return ipSrcAddr; + } + + public void setIpSrcAddr(String ipSrcAddr) { + this.ipSrcAddr = ipSrcAddr; + } + + public String getIpDstAddr() { + return ipDstAddr; + } + + public void setIpDstAddr(String ipDstAddr) { + this.ipDstAddr = ipDstAddr; + } + + public Integer getIpSrcPort() { + return ipSrcPort; + } + + public void setIpSrcPort(Integer ipSrcPort) { + this.ipSrcPort = ipSrcPort; + } + + public Integer getIpDstPort() { + return ipDstPort; + } + + public void setIpDstPort(Integer ipDstPort) { + this.ipDstPort = ipDstPort; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public String getPacketFilter() { + return packetFilter; + } + + public void setPacketFilter(String packetFilter) { + this.packetFilter = packetFilter; + } + + public Boolean getIncludeReverse() { + return includeReverse; + } + + public void setIncludeReverse(Boolean includeReverse) { + this.includeReverse = includeReverse; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FixedPcapRequest fixedPcapRequest = (FixedPcapRequest) o; + + return (super.equals(o)) && + (getIpSrcAddr() != null ? getIpSrcAddr().equals(fixedPcapRequest.getIpSrcAddr()) : fixedPcapRequest.getIpSrcAddr() != null) && + (getIpDstAddr() != null ? getIpDstAddr().equals(fixedPcapRequest.getIpDstAddr()) : fixedPcapRequest.getIpDstAddr() != null) && + (getIpSrcPort() != null ? getIpSrcPort().equals(fixedPcapRequest.getIpSrcPort()) : fixedPcapRequest.getIpSrcPort() != null) && + (getIpDstPort() != null ? getIpDstPort().equals(fixedPcapRequest.getIpDstPort()) : fixedPcapRequest.getIpDstPort() != null) && + (getProtocol() != null ? getProtocol().equals(fixedPcapRequest.getProtocol()) : fixedPcapRequest.getProtocol() != null) && + (getPacketFilter() != null ? getPacketFilter().equals(fixedPcapRequest.getPacketFilter()) : fixedPcapRequest.getPacketFilter() != null) && + (getIncludeReverse() != null ? getIncludeReverse().equals(fixedPcapRequest.getIncludeReverse()) : fixedPcapRequest.getIncludeReverse() != null); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (getIpSrcAddr() != null ? getIpSrcAddr().hashCode() : 0); + result = 31 * result + (getIpDstAddr() != null ? getIpDstAddr().hashCode() : 0); + result = 31 * result + (getIpSrcPort() != null ? getIpSrcPort().hashCode() : 0); + result = 31 * result + (getIpDstPort() != null ? getIpDstPort().hashCode() : 0); + result = 31 * result + (getProtocol() != null ? getProtocol().hashCode() : 0); + result = 31 * result + (getPacketFilter() != null ? getPacketFilter().hashCode() : 0); + result = 31 * result + (getIncludeReverse() != null ? getIncludeReverse().hashCode() : 0); + return result; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java new file mode 100644 index 0000000..6062f94 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model.pcap; + +public class PcapRequest { + + private String baseOutputPath; + private String basePath; + private Long startTime = 0L; + private Long endTime = System.currentTimeMillis(); + private Integer numReducers = 1; + + public String getBaseOutputPath() { + return baseOutputPath; + } + + public void setBaseOutputPath(String baseOutputPath) { + this.baseOutputPath = baseOutputPath; + } + + public String getBasePath() { + return basePath; + } + + public void setBasePath(String basePath) { + this.basePath = basePath; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public Long getEndTime() { + return endTime; + } + + public void setEndTime(Long endTime) { + this.endTime = endTime; + } + + public Integer getNumReducers() { + return numReducers; + } + + public void setNumReducers(Integer numReducers) { + this.numReducers = numReducers; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + PcapRequest pcapRequest = (PcapRequest) o; + + return (getBaseOutputPath() != null ? getBaseOutputPath().equals(pcapRequest.getBaseOutputPath()) : pcapRequest.getBaseOutputPath() != null) && + (getBasePath() != null ? getBasePath().equals(pcapRequest.getBasePath()) : pcapRequest.getBasePath() == null) && + (getStartTime() != null ? getStartTime().equals(pcapRequest.getStartTime()) : pcapRequest.getStartTime() == null) && + (getEndTime() != null ? getEndTime().equals(pcapRequest.getEndTime()) : pcapRequest.getEndTime() == null) && + (getNumReducers() != null ? getNumReducers().equals(pcapRequest.getNumReducers()) : pcapRequest.getNumReducers() == null); + } + + @Override + public int hashCode() { + int result = getBaseOutputPath() != null ? getBaseOutputPath().hashCode() : 0; + result = 31 * result + (getBasePath() != null ? getBasePath().hashCode() : 0); + result = 31 * result + (getStartTime() != null ? getStartTime().hashCode() : 0); + result = 31 * result + (getEndTime() != null ? getEndTime().hashCode() : 0); + result = 31 * result + (getNumReducers() != null ? getNumReducers().hashCode() : 0); + return result; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 2a6a0e0..016c6f9 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -241,6 +241,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)| | [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)| | [ `GET /api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)| +| [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)| | [ `GET /api/v1/search/search`](#get-apiv1searchsearch)| | [ `POST /api/v1/search/search`](#get-apiv1searchsearch)| | [ `POST /api/v1/search/group`](#get-apiv1searchgroup)| @@ -485,6 +486,13 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Returns: * 200 - Returns 'true' if the status changed and 'false' if it did not. +### `POST /api/v1/pcap/fixed` + * Description: Executes a Fixed Pcap Query. + * Input: + * fixedPcapRequest - A Fixed Pcap Request which includes fixed filter fields like ip source address and protocol. + * Returns: + * 200 - Returns a PcapResponse containing an array of pcaps. + ### `POST /api/v1/search/search` * Description: Searches the indexing store. GUIDs must be quoted to ensure correct results. * Input: http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 9c9c372..6e9553e 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -170,6 +170,28 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <version>${global_hadoop_version}</version> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <version>${global_hadoop_version}</version> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>org.apache.metron</groupId> <artifactId>metron-hbase-client</artifactId> @@ -372,6 +394,17 @@ <artifactId>org.eclipse.persistence.jpa</artifactId> <version>${eclipse.link.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-pcap</artifactId> + <version>${project.parent.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> <dependencyManagement> http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index c4873f9..0989d12 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -74,4 +74,7 @@ public class MetronRestConstants { public static final String USER_SETTINGS_HBASE_CF_SPRING_PROPERTY = "user.settings.cf"; public static final String LOGGING_SYSTEM_PROPERTY = "org.springframework.boot.logging.LoggingSystem"; + + public static final String PCAP_INPUT_PATH_SPRING_PROPERTY = "pcap.input.path"; + public static final String PCAP_OUTPUT_PATH_SPRING_PROPERTY = "pcap.output.path"; } http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java new file mode 100644 index 0000000..8da5f96 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import org.apache.metron.pcap.mr.PcapJob; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; + +@Configuration +@Profile("!" + TEST_PROFILE) +public class PcapConfig { + + @Bean + public PcapJob pcapJob() { + return new PcapJob(); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java new file mode 100644 index 0000000..11ff1cd --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.metron.rest.model.PcapResponse; +import org.apache.metron.rest.model.pcap.FixedPcapRequest; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.PcapService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; + +@RestController +@RequestMapping("/api/v1/pcap") +public class PcapController { + + @Autowired + private PcapService pcapQueryService; + + @ApiOperation(value = "Executes a Fixed Pcap Query.") + @ApiResponses(value = { @ApiResponse(message = "Returns a PcapResponse containing an array of pcaps.", code = 200)}) + @RequestMapping(value = "/fixed", method = RequestMethod.POST) + ResponseEntity<PcapResponse> fixed(@ApiParam(name="fixedPcapRequest", value="A Fixed Pcap Request" + + " which includes fixed filter fields like ip source address and protocol.", required=true)@RequestBody FixedPcapRequest fixedPcapRequest) throws RestException { + PcapResponse pcapsResponse = pcapQueryService.fixed(fixedPcapRequest); + return new ResponseEntity<>(pcapsResponse, HttpStatus.OK); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java new file mode 100644 index 0000000..ce8372c --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.PcapResponse; +import org.apache.metron.rest.model.pcap.FixedPcapRequest; + +public interface PcapService { + + PcapResponse fixed(FixedPcapRequest fixedPcapRequest) throws RestException; +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java new file mode 100644 index 0000000..4dae1e5 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.metron.common.Constants; +import org.apache.metron.common.hadoop.SequenceFileIterable; +import org.apache.metron.common.utils.timestamp.TimestampConverters; +import org.apache.metron.pcap.PcapHelper; +import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; +import org.apache.metron.pcap.mr.PcapJob; +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.PcapResponse; +import org.apache.metron.rest.model.pcap.FixedPcapRequest; +import org.apache.metron.rest.service.PcapService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class PcapServiceImpl implements PcapService { + + private Environment environment; + private Configuration configuration; + private PcapJob pcapJob; + + @Autowired + public PcapServiceImpl(Environment environment, Configuration configuration, PcapJob pcapJob) { + this.environment = environment; + this.configuration = configuration; + this.pcapJob = pcapJob; + } + + @Override + public PcapResponse fixed(FixedPcapRequest fixedPcapRequest) throws RestException { + if (fixedPcapRequest.getBasePath() == null) { + fixedPcapRequest.setBasePath(environment.getProperty(MetronRestConstants.PCAP_INPUT_PATH_SPRING_PROPERTY)); + } + if (fixedPcapRequest.getBaseOutputPath() == null) { + fixedPcapRequest.setBaseOutputPath(environment.getProperty(MetronRestConstants.PCAP_OUTPUT_PATH_SPRING_PROPERTY)); + } + PcapResponse response = new PcapResponse(); + SequenceFileIterable results; + try { + results = pcapJob.query( + new Path(fixedPcapRequest.getBasePath()), + new Path(fixedPcapRequest.getBaseOutputPath()), + TimestampConverters.MILLISECONDS.toNanoseconds(fixedPcapRequest.getStartTime()), + TimestampConverters.MILLISECONDS.toNanoseconds(fixedPcapRequest.getEndTime()), + fixedPcapRequest.getNumReducers(), + getFixedFields(fixedPcapRequest), + configuration, + getFileSystem(), + new FixedPcapFilter.Configurator() + ); + if (results != null) { + List<byte[]> pcaps = new ArrayList<>(); + results.iterator().forEachRemaining(pcaps::add); + response.setPcaps(pcaps); + } + } catch (IOException | ClassNotFoundException | InterruptedException e) { + throw new RestException(e); + } + return response; + } + + protected Map<String, String> getFixedFields(FixedPcapRequest fixedPcapRequest) { + Map<String, String> fixedFields = new HashMap<>(); + if (fixedPcapRequest.getIpSrcAddr() != null) { + fixedFields.put(Constants.Fields.SRC_ADDR.getName(), fixedPcapRequest.getIpSrcAddr()); + } + if (fixedPcapRequest.getIpDstAddr() != null) { + fixedFields.put(Constants.Fields.DST_ADDR.getName(), fixedPcapRequest.getIpDstAddr()); + } + if (fixedPcapRequest.getIpSrcPort() != null) { + fixedFields.put(Constants.Fields.SRC_PORT.getName(), fixedPcapRequest.getIpSrcPort().toString()); + } + if (fixedPcapRequest.getIpDstPort() != null) { + fixedFields.put(Constants.Fields.DST_PORT.getName(), fixedPcapRequest.getIpDstPort().toString()); + } + if (fixedPcapRequest.getProtocol() != null) { + fixedFields.put(Constants.Fields.PROTOCOL.getName(), fixedPcapRequest.getProtocol()); + } + if (fixedPcapRequest.getIncludeReverse() != null) { + fixedFields.put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), fixedPcapRequest.getIncludeReverse().toString()); + } + if (fixedPcapRequest.getPacketFilter() != null) { + fixedFields.put(PcapHelper.PacketFields.PACKET_FILTER.getName(), fixedPcapRequest.getPacketFilter()); + } + return fixedFields; + } + + protected FileSystem getFileSystem() throws IOException { + return FileSystem.get(configuration); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/resources/application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml index 97359b9..df67ab3 100644 --- a/metron-interface/metron-rest/src/main/resources/application.yml +++ b/metron-interface/metron-rest/src/main/resources/application.yml @@ -71,4 +71,8 @@ spring.jpa.generate-ddl: true user: settings: table: user_settings - cf: cf \ No newline at end of file + cf: cf + +pcap: + input.path: /apps/metron/pcap + output.path: /tmp \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/scripts/metron-rest.sh ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh index 2c7c75b..cf7bf58 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh @@ -110,7 +110,7 @@ fi echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}" echo "Starting application" -${JAVA_HOME}/bin/java ${METRON_JVMFLAGS} \ +${JAVA_HOME}/bin/java -Dhdp.version=${HDP_VERSION} ${METRON_JVMFLAGS} \ -cp ${METRON_REST_CLASSPATH} \ org.apache.metron.rest.MetronRestApplication \ ${METRON_SPRING_OPTIONS} >> ${METRON_LOG_DIR}/metron-rest.log 2>&1 & echo $! > ${METRON_PID_FILE}; http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java index 008f3fc..a9e70d2 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java @@ -37,7 +37,9 @@ import org.apache.metron.integration.ComponentRunner; import org.apache.metron.integration.UnableToStartException; import org.apache.metron.integration.components.KafkaComponent; import org.apache.metron.integration.components.ZKServerComponent; +import org.apache.metron.pcap.mr.PcapJob; import org.apache.metron.rest.RestException; +import org.apache.metron.rest.mock.MockPcapJob; import org.apache.metron.rest.mock.MockStormCLIClientWrapper; import org.apache.metron.rest.mock.MockStormRestTemplate; import org.apache.metron.rest.service.impl.StormCLIWrapper; @@ -185,4 +187,9 @@ public class TestConfig { public UserSettingsClient userSettingsClient() throws RestException, IOException { return new UserSettingsClient(new MockHBaseTableProvider().addToCache("user_settings", "cf"), Bytes.toBytes("cf")); } + + @Bean + public PcapJob mockPcapJob() { + return new MockPcapJob(); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java new file mode 100644 index 0000000..5e4875a --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.Constants; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.pcap.PcapHelper; +import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; +import org.apache.metron.rest.mock.MockPcapJob; +import org.apache.metron.rest.model.PcapResponse; +import org.apache.metron.rest.service.PcapService; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class PcapControllerIntegrationTest { + + /** + { + "basePath": "/apps/metron/pcap", + "baseOutputPath": "/tmp", + "endTime": 10, + "includeReverse": "true", + "ipDstAddr": "192.168.1.1", + "ipDstPort": "1000", + "ipSrcAddr": "192.168.1.2", + "ipSrcPort": "2000", + "numReducers": 2, + "packetFilter": "filter", + "protocol": "TCP", + "startTime": 1 + } + */ + @Multiline + public static String fixedJson; + + @Autowired + private PcapService pcapService; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String pcapUrl = "/api/v1/pcap"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(pcapUrl + "/fixed").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) + .andExpect(status().isUnauthorized()); + } + + @Test + public void testFixed() throws Exception { + MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); + List<byte[]> results = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes()); + mockPcapJob.setResults(results); + + PcapResponse expectedReponse = new PcapResponse(); + expectedReponse.setPcaps(results); + this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(content().json(JSONUtils.INSTANCE.toJSON(expectedReponse, false))); + + Assert.assertEquals("/apps/metron/pcap", mockPcapJob.getBasePath()); + Assert.assertEquals("/tmp", mockPcapJob.getBaseOutputPath()); + Assert.assertEquals(1, mockPcapJob.getStartTime()); + Assert.assertEquals(10, mockPcapJob.getEndTime()); + Assert.assertEquals(2, mockPcapJob.getNumReducers()); + Assert.assertTrue(mockPcapJob.getFilterImpl() instanceof FixedPcapFilter.Configurator); + Map<String, String> actualFixedFields = mockPcapJob.getFixedFields(); + Assert.assertEquals("192.168.1.2", actualFixedFields.get(Constants.Fields.SRC_ADDR.getName())); + Assert.assertEquals("2000", actualFixedFields.get(Constants.Fields.SRC_PORT.getName())); + Assert.assertEquals("192.168.1.1", actualFixedFields.get(Constants.Fields.DST_ADDR.getName())); + Assert.assertEquals("1000", actualFixedFields.get(Constants.Fields.DST_PORT.getName())); + Assert.assertEquals("true", actualFixedFields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName())); + Assert.assertEquals("TCP", actualFixedFields.get(Constants.Fields.PROTOCOL.getName())); + Assert.assertEquals("filter", actualFixedFields.get(PcapHelper.PacketFields.PACKET_FILTER.getName())); + + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java new file mode 100644 index 0000000..3aa9ce3 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.mock; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.metron.common.hadoop.SequenceFileIterable; +import org.apache.metron.pcap.filter.PcapFilterConfigurator; +import org.apache.metron.pcap.mr.PcapJob; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.mockito.Matchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MockPcapJob extends PcapJob { + + private String basePath; + private String baseOutputPath; + private long beginNS; + private long endNS; + private int numReducers; + private Map<String, String> fixedFields; + private PcapFilterConfigurator filterImpl; + private SequenceFileIterable sequenceFileIterable; + + public MockPcapJob() { + sequenceFileIterable = mock(SequenceFileIterable.class); + } + + @SuppressWarnings(value = "unchecked") + @Override + public <T> SequenceFileIterable query(Path basePath, Path baseOutputPath, long beginNS, long endNS, int numReducers, T fields, Configuration conf, FileSystem fs, PcapFilterConfigurator<T> filterImpl) throws IOException, ClassNotFoundException, InterruptedException { + this.basePath = basePath.toString(); + this.baseOutputPath = baseOutputPath.toString(); + this.beginNS = beginNS; + this.endNS = endNS; + this.numReducers = numReducers; + if (fields instanceof Map) { + this.fixedFields = (Map<String, String>) fields; + } + this.filterImpl = filterImpl; + return sequenceFileIterable; + } + + public void setResults(List<byte[]> pcaps) { + when(sequenceFileIterable.iterator()).thenReturn(pcaps.iterator()); + } + + public String getBasePath() { + return basePath; + } + + public String getBaseOutputPath() { + return baseOutputPath; + } + + public long getStartTime() { + return beginNS / 1000000; + } + + public long getEndTime() { + return endNS / 1000000; + } + + public int getNumReducers() { + return numReducers; + } + + public Map<String, String> getFixedFields() { + return fixedFields; + } + + public PcapFilterConfigurator getFilterImpl() { + return filterImpl; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java new file mode 100644 index 0000000..b6f2ea8 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.metron.common.Constants; +import org.apache.metron.common.hadoop.SequenceFileIterable; +import org.apache.metron.pcap.PcapHelper; +import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; +import org.apache.metron.pcap.mr.PcapJob; +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.PcapResponse; +import org.apache.metron.rest.model.pcap.FixedPcapRequest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.springframework.core.env.Environment; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@SuppressWarnings("ALL") +public class PcapServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + Environment environment; + Configuration configuration; + PcapJob pcapJob; + + @Before + public void setUp() throws Exception { + environment = mock(Environment.class); + pcapJob = mock(PcapJob.class); + configuration = mock(Configuration.class); + + when(environment.getProperty(MetronRestConstants.PCAP_INPUT_PATH_SPRING_PROPERTY)).thenReturn("/input/path"); + when(environment.getProperty(MetronRestConstants.PCAP_OUTPUT_PATH_SPRING_PROPERTY)).thenReturn("/output/path"); + } + + + @Test + public void fixedShouldProperlyCallPcapJobQuery() throws Exception { + FixedPcapRequest fixedPcapRequest = new FixedPcapRequest(); + fixedPcapRequest.setBaseOutputPath("baseOutputPath"); + fixedPcapRequest.setBasePath("basePath"); + fixedPcapRequest.setStartTime(1L); + fixedPcapRequest.setEndTime(2L); + fixedPcapRequest.setNumReducers(2); + fixedPcapRequest.setIpSrcAddr("ip_src_addr"); + fixedPcapRequest.setIpDstAddr("ip_dst_addr"); + fixedPcapRequest.setIpSrcPort(1000); + fixedPcapRequest.setIpDstPort(2000); + fixedPcapRequest.setProtocol("tcp"); + fixedPcapRequest.setPacketFilter("filter"); + fixedPcapRequest.setIncludeReverse(true); + + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, pcapJob)); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + Map<String, String> expectedFields = new HashMap<String, String>() {{ + put(Constants.Fields.SRC_ADDR.getName(), "ip_src_addr"); + put(Constants.Fields.DST_ADDR.getName(), "ip_dst_addr"); + put(Constants.Fields.SRC_PORT.getName(), "1000"); + put(Constants.Fields.DST_PORT.getName(), "2000"); + put(Constants.Fields.PROTOCOL.getName(), "tcp"); + put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true"); + put(PcapHelper.PacketFields.PACKET_FILTER.getName(), "filter"); + }}; + List<byte[]> expectedPcaps = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes()); + SequenceFileIterable results = mock(SequenceFileIterable.class); + when(results.iterator()).thenReturn(expectedPcaps.iterator()); + when(pcapJob.query(eq(new Path("basePath")), + eq(new Path("baseOutputPath")), + eq(1000000L), + eq(2000000L), + eq(2), + eq(expectedFields), + eq(configuration), + any(FileSystem.class), + any(FixedPcapFilter.Configurator.class))).thenReturn(results); + + PcapResponse pcapsResponse = pcapService.fixed(fixedPcapRequest); + Assert.assertEquals(expectedPcaps, pcapsResponse.getPcaps()); + } + + @Test + public void fixedShouldProperlyCallPcapJobQueryWithDefaults() throws Exception { + FixedPcapRequest fixedPcapRequest = new FixedPcapRequest(); + + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, pcapJob)); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + Map<String, String> expectedFields = new HashMap<String, String>() {{ + put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "false"); + }}; + List<byte[]> expectedPcaps = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes()); + SequenceFileIterable results = mock(SequenceFileIterable.class); + when(results.iterator()).thenReturn(expectedPcaps.iterator()); + when(pcapJob.query(eq(new Path("/input/path")), + eq(new Path("/output/path")), + eq(0L), + eq(fixedPcapRequest.getEndTime() * 1000000), + eq(1), + eq(expectedFields), + eq(configuration), + any(FileSystem.class), + any(FixedPcapFilter.Configurator.class))).thenReturn(results); + + PcapResponse pcapsResponse = pcapService.fixed(fixedPcapRequest); + Assert.assertEquals(expectedPcaps, pcapsResponse.getPcaps()); + } + + @Test + public void fixedShouldThrowRestException() throws Exception { + exception.expect(RestException.class); + exception.expectMessage("some exception"); + + FixedPcapRequest fixedPcapRequest = new FixedPcapRequest(); + + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, pcapJob)); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + + when(pcapJob.query(any(), + any(), + eq(0L), + eq(fixedPcapRequest.getEndTime() * 1000000), + eq(1), + any(), + any(), + any(FileSystem.class), + any(FixedPcapFilter.Configurator.class))).thenThrow(new IOException("some exception")); + + pcapService.fixed(fixedPcapRequest); + } +}
