METRON-1691: REST should limit the number of Pcap jobs a user can submit (merrimanr via mmiklavc) closes apache/metron#1129
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6c90724d Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6c90724d Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6c90724d Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: 6c90724d8d4f06c453128f860eff51037f1870d8 Parents: f1f5dda Author: merrimanr <merrim...@gmail.com> Authored: Wed Jul 25 10:58:05 2018 -0600 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Wed Jul 25 10:58:05 2018 -0600 ---------------------------------------------------------------------- .../apache/metron/rest/model/pcap/Field.java | 36 ++++------- .../rest/model/pcap/FixedPcapRequest.java | 30 --------- .../apache/metron/rest/model/pcap/Packet.java | 8 +-- .../metron/rest/model/pcap/PcapStatus.java | 24 +++---- .../org/apache/metron/rest/model/pcap/Pdml.java | 21 +++--- .../apache/metron/rest/model/pcap/Proto.java | 24 +++---- .../rest/model/pcap/QueryPcapRequest.java | 18 ------ metron-interface/metron-rest/README.md | 27 ++++++-- .../apache/metron/rest/MetronRestConstants.java | 1 + .../metron/rest/controller/PcapController.java | 13 ++++ .../apache/metron/rest/service/PcapService.java | 6 ++ .../rest/service/impl/PcapServiceImpl.java | 64 ++++++++++++++++-- .../src/main/resources/application.yml | 1 + .../PcapControllerIntegrationTest.java | 46 +++++++++++++ .../rest/service/impl/PcapServiceImplTest.java | 68 ++++++++++++++++++++ .../metron/job/manager/InMemoryJobManager.java | 4 ++ 16 files changed, 261 insertions(+), 130 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java index 9c2878b..4ed71c3 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class Field { @@ -132,33 +133,22 @@ public class Field { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Field field = (Field) o; - - return (getName() != null ? getName().equals(field.getName()) : field.getName() != null) && - (getPos() != null ? getPos().equals(field.getPos()) : field.getPos() == null) && - (getShowname() != null ? getShowname().equals(field.getShowname()) : field.getShowname() == null) && - (getSize() != null ? getSize().equals(field.getSize()) : field.getSize() == null) && - (getValue() != null ? getValue().equals(field.getValue()) : field.getValue() == null) && - (getShow() != null ? getShow().equals(field.getShow()) : field.getShow() == null) && - (getUnmaskedvalue() != null ? getUnmaskedvalue().equals(field.getUnmaskedvalue()) : field.getUnmaskedvalue() == null) && - (getHide() != null ? getHide().equals(field.getHide()) : field.getHide() == null) && - (getFields() != null ? getFields().equals(field.getFields()) : field.getFields() == null) && - (getProtos() != null ? getProtos().equals(field.getProtos()) : field.getProtos() == null); + return Objects.equals(name, field.name) && + Objects.equals(pos, field.pos) && + Objects.equals(showname, field.showname) && + Objects.equals(size, field.size) && + Objects.equals(value, field.value) && + Objects.equals(show, field.show) && + Objects.equals(unmaskedvalue, field.unmaskedvalue) && + Objects.equals(hide, field.hide) && + Objects.equals(fields, field.fields) && + Objects.equals(protos, field.protos); } @Override public int hashCode() { - int result = getName() != null ? getName().hashCode() : 0; - result = 31 * result + (getPos() != null ? getPos().hashCode() : 0); - result = 31 * result + (getShowname() != null ? getShowname().hashCode() : 0); - result = 31 * result + (getSize() != null ? getSize().hashCode() : 0); - result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); - result = 31 * result + (getShow() != null ? getShow().hashCode() : 0); - result = 31 * result + (getUnmaskedvalue() != null ? getUnmaskedvalue().hashCode() : 0); - result = 31 * result + (getHide() != null ? getHide().hashCode() : 0); - result = 31 * result + (getFields() != null ? getFields().hashCode() : 0); - result = 31 * result + (getProtos() != null ? getProtos().hashCode() : 0); - return result; + + return Objects.hash(name, pos, showname, size, value, show, unmaskedvalue, hide, fields, protos); } } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java index d91aac7..38d05b7 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java @@ -113,34 +113,4 @@ public class FixedPcapRequest extends PcapRequest { } PcapOptions.FIELDS.put(this, fields); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - FixedPcapRequest fixedPcapRequest = (FixedPcapRequest) o; - - return (super.equals(o)) && - (getIpSrcAddr() != null ? getIpSrcAddr().equals(fixedPcapRequest.getIpSrcAddr()) : fixedPcapRequest.getIpSrcAddr() != null) && - (getIpDstAddr() != null ? getIpDstAddr().equals(fixedPcapRequest.getIpDstAddr()) : fixedPcapRequest.getIpDstAddr() != null) && - (getIpSrcPort() != null ? getIpSrcPort().equals(fixedPcapRequest.getIpSrcPort()) : fixedPcapRequest.getIpSrcPort() != null) && - (getIpDstPort() != null ? getIpDstPort().equals(fixedPcapRequest.getIpDstPort()) : fixedPcapRequest.getIpDstPort() != null) && - (getProtocol() != null ? getProtocol().equals(fixedPcapRequest.getProtocol()) : fixedPcapRequest.getProtocol() != null) && - (getPacketFilter() != null ? getPacketFilter().equals(fixedPcapRequest.getPacketFilter()) : fixedPcapRequest.getPacketFilter() != null) && - (getIncludeReverse() != null ? getIncludeReverse().equals(fixedPcapRequest.getIncludeReverse()) : fixedPcapRequest.getIncludeReverse() != null); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (getIpSrcAddr() != null ? getIpSrcAddr().hashCode() : 0); - result = 31 * result + (getIpDstAddr() != null ? getIpDstAddr().hashCode() : 0); - result = 31 * result + (getIpSrcPort() != null ? getIpSrcPort().hashCode() : 0); - result = 31 * result + (getIpDstPort() != null ? getIpDstPort().hashCode() : 0); - result = 31 * result + (getProtocol() != null ? getProtocol().hashCode() : 0); - result = 31 * result + (getPacketFilter() != null ? getPacketFilter().hashCode() : 0); - result = 31 * result + (getIncludeReverse() != null ? getIncludeReverse().hashCode() : 0); - return result; - } } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java index de21e6b..1773272 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.util.List; +import java.util.Objects; public class Packet { @@ -40,14 +41,13 @@ public class Packet { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Packet packet = (Packet) o; - - return (getProtos() != null ? getProtos().equals(packet.getProtos()) : packet.getProtos() == null); + return Objects.equals(protos, packet.protos); } @Override public int hashCode() { - return getProtos() != null ? getProtos().hashCode() : 0; + + return Objects.hash(protos); } } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java index f004eb5..43c77fd 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java @@ -17,6 +17,8 @@ */ package org.apache.metron.rest.model.pcap; +import java.util.Objects; + public class PcapStatus { private String jobId; @@ -69,23 +71,17 @@ public class PcapStatus { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - - PcapStatus pcapStatus = (PcapStatus) o; - - return (getJobId() != null ? getJobId().equals(pcapStatus.getJobId()) : pcapStatus.getJobId() != null) && - (getJobStatus() != null ? getJobStatus().equals(pcapStatus.getJobStatus()) : pcapStatus.getJobStatus() != null) && - (getDescription() != null ? getDescription().equals(pcapStatus.getDescription()) : pcapStatus.getDescription() != null) && - (getPercentComplete() != null ? getPercentComplete().equals(pcapStatus.getPercentComplete()) : pcapStatus.getPercentComplete() != null) && - (getPageTotal() != null ? getPageTotal().equals(pcapStatus.getPageTotal()) : pcapStatus.getPageTotal() != null); + PcapStatus that = (PcapStatus) o; + return Objects.equals(jobId, that.jobId) && + Objects.equals(jobStatus, that.jobStatus) && + Objects.equals(description, that.description) && + Objects.equals(percentComplete, that.percentComplete) && + Objects.equals(pageTotal, that.pageTotal); } @Override public int hashCode() { - int result = (getJobId() != null ? getJobId().hashCode() : 0); - result = 31 * result + (getJobStatus() != null ? getJobStatus().hashCode() : 0); - result = 31 * result + (getDescription() != null ? getDescription().hashCode() : 0); - result = 31 * result + (getPercentComplete() != null ? getPercentComplete().hashCode() : 0); - result = 31 * result + (getPageTotal() != null ? getPageTotal().hashCode() : 0); - return result; + + return Objects.hash(jobId, jobStatus, description, percentComplete, pageTotal); } } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java index f44f96b..f59586a 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.util.List; +import java.util.Objects; public class Pdml { @@ -81,23 +82,17 @@ public class Pdml { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Pdml pdml = (Pdml) o; - - return (getVersion() != null ? getVersion().equals(pdml.getVersion()) : pdml.getVersion() != null) && - (getCreator() != null ? getCreator().equals(pdml.getCreator()) : pdml.getCreator() == null) && - (getTime() != null ? getTime().equals(pdml.getTime()) : pdml.getTime() == null) && - (getCaptureFile() != null ? getCaptureFile().equals(pdml.getCaptureFile()) : pdml.getCaptureFile() == null) && - (getPackets() != null ? getPackets().equals(pdml.getPackets()) : pdml.getPackets() == null); + return Objects.equals(version, pdml.version) && + Objects.equals(creator, pdml.creator) && + Objects.equals(time, pdml.time) && + Objects.equals(captureFile, pdml.captureFile) && + Objects.equals(packets, pdml.packets); } @Override public int hashCode() { - int result = getVersion() != null ? getVersion().hashCode() : 0; - result = 31 * result + (getCreator() != null ? getCreator().hashCode() : 0); - result = 31 * result + (getTime() != null ? getTime().hashCode() : 0); - result = 31 * result + (getCaptureFile() != null ? getCaptureFile().hashCode() : 0); - result = 31 * result + (getPackets() != null ? getPackets().hashCode() : 0); - return result; + + return Objects.hash(version, creator, time, captureFile, packets); } } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java index bdd5c1f..2c145a3 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import java.util.List; +import java.util.Objects; public class Proto { @@ -90,25 +91,18 @@ public class Proto { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Proto proto = (Proto) o; - - return (getName() != null ? getName().equals(proto.getName()) : proto.getName() != null) && - (getPos() != null ? getPos().equals(proto.getPos()) : proto.getPos() == null) && - (getShowname() != null ? getShowname().equals(proto.getShowname()) : proto.getShowname() == null) && - (getSize() != null ? getSize().equals(proto.getSize()) : proto.getSize() == null) && - (getHide() != null ? getHide().equals(proto.getHide()) : proto.getHide() == null) && - (getFields() != null ? getFields().equals(proto.getFields()) : proto.getFields() == null); + return Objects.equals(name, proto.name) && + Objects.equals(pos, proto.pos) && + Objects.equals(showname, proto.showname) && + Objects.equals(size, proto.size) && + Objects.equals(hide, proto.hide) && + Objects.equals(fields, proto.fields); } @Override public int hashCode() { - int result = getName() != null ? getName().hashCode() : 0; - result = 31 * result + (getPos() != null ? getPos().hashCode() : 0); - result = 31 * result + (getShowname() != null ? getShowname().hashCode() : 0); - result = 31 * result + (getSize() != null ? getSize().hashCode() : 0); - result = 31 * result + (getHide() != null ? getHide().hashCode() : 0); - result = 31 * result + (getFields() != null ? getFields().hashCode() : 0); - return result; + + return Objects.hash(name, pos, showname, size, hide, fields); } } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java index 0da3e69..1a03fda 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java @@ -38,22 +38,4 @@ public class QueryPcapRequest extends PcapRequest { public void setFields() { PcapOptions.FIELDS.put(this, getQuery()); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - QueryPcapRequest queryPcapRequest = (QueryPcapRequest) o; - - return (super.equals(o)) && - (getQuery() != null ? getQuery().equals(queryPcapRequest.getQuery()) : queryPcapRequest.getQuery() != null); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (getQuery() != null ? getQuery().hashCode() : 0); - return result; - } } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 53f5978..2d9a535 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -212,7 +212,7 @@ METRON_SPRING_PROFILES_ACTIVE="vagrant,dev" ## Pcap Query -The REST application exposes endpoints for querying Pcap data. For more information about filtering options see [Query Filter Utility](/metron-platform/metron-pcap-backend#query-filter-utility). +The REST application exposes endpoints for querying Pcap data. For more information about filtering options see [Query Filter Utility](../../metron-platform/metron-pcap-backend#query-filter-utility). There is an endpoint available that will return Pcap data in [PDML](https://wiki.wireshark.org/PDML) format. [Wireshark](https://www.wireshark.org/) must be installed for this feature to work. Installing wireshark in CentOS can be done with `yum -y install wireshark`. @@ -253,8 +253,9 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)| | [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)| | [ `GET /api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)| -| [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)| -| [ `GET /api/v1/pcap/query`](#get-apiv1pcapquery)| +| [ `POST /api/v1/pcap/fixed`](#post-apiv1pcapfixed)| +| [ `POST /api/v1/pcap/query`](#post-apiv1pcapquery)| +| [ `GET /api/v1/pcap`](#get-apiv1pcap)| | [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)| | [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)| | [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)| @@ -516,7 +517,14 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Returns: * 200 - Returns a job status with job ID. -### `POST /api/v1/pcap/{jobId}` +### `GET /api/v1/pcap` + * Description: Gets a list of job statuses for Pcap query jobs that match the requested state. + * Input: + * state - Job state + * Returns: + * 200 - Returns a list of job statuses for jobs that match the requested state. + +### `GET /api/v1/pcap/{jobId}` * Description: Gets job status for Pcap query job. * Input: * jobId - Job ID of submitted job @@ -524,7 +532,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns a job status for the Job ID. * 404 - Job is missing. -### `POST /api/v1/pcap/{jobId}/pdml` +### `GET /api/v1/pcap/{jobId}/pdml` * Description: Gets Pcap Results for a page in PDML format. * Input: * jobId - Job ID of submitted job @@ -533,7 +541,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns PDML in json format. * 404 - Job or page is missing. -### `POST /api/v1/pcap/{jobId}/raw` +### `GET /api/v1/pcap/{jobId}/raw` * Description: Download Pcap Results for a page. * Input: * jobId - Job ID of submitted job @@ -541,6 +549,13 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Returns: * 200 - Returns Pcap as a file download. * 404 - Job or page is missing. + +### `DELETE /api/v1/pcap/kill/{jobId}` + * Description: Kills running job. + * Input: + * jobId - Job ID of submitted job + * Returns: + * 200 - Kills passed job. ### `POST /api/v1/search/search` * Description: Searches the indexing store. GUIDs must be quoted to ensure correct results. http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index b65d037..d38aedb 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -72,6 +72,7 @@ public class MetronRestConstants { public static final String USER_SETTINGS_HBASE_TABLE_SPRING_PROPERTY = "user.settings.table"; public static final String USER_SETTINGS_HBASE_CF_SPRING_PROPERTY = "user.settings.cf"; + public static final String USER_JOB_LIMIT_SPRING_PROPERTY = "user.job.limit"; public static final String LOGGING_SYSTEM_PROPERTY = "org.springframework.boot.logging.LoggingSystem"; http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java index be95718..13a623a 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import org.apache.commons.io.IOUtils; +import org.apache.metron.job.JobStatus; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.pcap.FixedPcapRequest; import org.apache.metron.rest.model.pcap.PcapStatus; @@ -44,6 +45,7 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.List; @RestController @RequestMapping("/api/v1/pcap") @@ -87,6 +89,17 @@ public class PcapController { } } + @ApiOperation(value = "Gets a list of job statuses for Pcap query jobs that match the requested state.") + @ApiResponses(value = { + @ApiResponse(message = "Returns a list of job statuses for jobs that match the requested state.", code = 200) + }) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity<List<PcapStatus>> getStatuses(@ApiParam(name="state", value="Job state", required=true)@RequestParam String state) throws RestException { + + List<PcapStatus> jobs = pcapQueryService.getJobStatus(SecurityUtils.getCurrentUser(), JobStatus.State.valueOf(state)); + return new ResponseEntity<>(jobs, HttpStatus.OK); + } + @ApiOperation(value = "Gets Pcap Results for a page in PDML format.") @ApiResponses(value = { @ApiResponse(message = "Returns PDML in json format.", code = 200), http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java index 2ae34a3..00efab9 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java @@ -18,12 +18,14 @@ package org.apache.metron.rest.service; import org.apache.hadoop.fs.Path; +import org.apache.metron.job.JobStatus; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.pcap.PcapRequest; import org.apache.metron.rest.model.pcap.PcapStatus; import org.apache.metron.rest.model.pcap.Pdml; import java.io.InputStream; +import java.util.List; public interface PcapService { @@ -31,6 +33,8 @@ public interface PcapService { PcapStatus getJobStatus(String username, String jobId) throws RestException; + List<PcapStatus> getJobStatus(String username, JobStatus.State state) throws RestException; + PcapStatus killJob(String username, String jobId) throws RestException; Path getPath(String username, String jobId, Integer page) throws RestException; @@ -38,4 +42,6 @@ public interface PcapService { Pdml getPdml(String username, String jobId, Integer page) throws RestException; InputStream getRawPcap(String username, String jobId, Integer page) throws RestException; + + } http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java index 77a8934..ff80c8f 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java @@ -41,6 +41,9 @@ import org.springframework.stereotype.Service; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; @Service public class PcapServiceImpl implements PcapService { @@ -62,6 +65,14 @@ public class PcapServiceImpl implements PcapService { @Override public PcapStatus submit(String username, PcapRequest pcapRequest) throws RestException { + List<PcapStatus> runningJobs = getJobStatus(username, JobStatus.State.RUNNING); + Integer userJobLimit = environment.getProperty(MetronRestConstants.USER_JOB_LIMIT_SPRING_PROPERTY, Integer.class, 1); + if (runningJobs != null && runningJobs.size() >= userJobLimit) { + String jobIds = runningJobs.stream().map(PcapStatus::getJobId).collect(Collectors.joining(", ")); + String message = String.format("Cannot submit job because a job is already running. " + + "Please contact the administrator to cancel job(s) with id(s) %s", jobIds); + throw new RestException(message); + } try { setPcapOptions(username, pcapRequest); pcapRequest.setFields(); @@ -79,13 +90,7 @@ public class PcapServiceImpl implements PcapService { try { Statusable<Path> statusable = jobManager.getJob(username, jobId); if (statusable != null) { - pcapStatus = jobStatusToPcapStatus(statusable.getStatus()); - if (statusable.isDone()) { - Pageable<Path> pageable = statusable.get(); - if (pageable != null) { - pcapStatus.setPageTotal(pageable.getSize()); - } - } + pcapStatus = statusableToPcapStatus(statusable); } } catch (JobNotFoundException | InterruptedException e) { // do nothing and return null pcapStatus @@ -96,6 +101,40 @@ public class PcapServiceImpl implements PcapService { } @Override + public List<PcapStatus> getJobStatus(String username, JobStatus.State state) throws RestException { + List<PcapStatus> pcapStatuses = new ArrayList<>(); + try { + List<Statusable<Path>> statusables = jobManager.getJobs(username); + if (statusables != null) { + pcapStatuses = statusables.stream() + .filter(statusable -> { + try { + return statusable.getStatus().getState() == state; + } catch (JobException e) { + return JobStatus.State.FAILED == state; + } + }) + .map(statusable -> { + try { + return statusableToPcapStatus(statusable); + } catch (JobException | InterruptedException e) { + PcapStatus pcapStatus = new PcapStatus(); + pcapStatus.setJobStatus(JobStatus.State.FAILED.toString()); + pcapStatus.setDescription(e.getMessage()); + return pcapStatus; + } + }) + .collect(Collectors.toList()); + } + } catch (JobNotFoundException e) { + // do nothing and return null pcapStatus + } catch (JobException e) { + throw new RestException(e); + } + return pcapStatuses; + } + + @Override public PcapStatus killJob(String username, String jobId) throws RestException { try { jobManager.killJob(username, jobId); @@ -183,6 +222,17 @@ public class PcapServiceImpl implements PcapService { return FileSystem.get(configuration); } + protected PcapStatus statusableToPcapStatus(Statusable<Path> statusable) throws JobException, InterruptedException { + PcapStatus pcapStatus = jobStatusToPcapStatus(statusable.getStatus()); + if (statusable.isDone()) { + Pageable<Path> pageable = statusable.get(); + if (pageable != null) { + pcapStatus.setPageTotal(pageable.getSize()); + } + } + return pcapStatus; + } + protected PcapStatus jobStatusToPcapStatus(JobStatus jobStatus) { PcapStatus pcapStatus = new PcapStatus(); pcapStatus.setJobId(jobStatus.getJobId()); http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/resources/application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml index 5fd9d72..866109e 100644 --- a/metron-interface/metron-rest/src/main/resources/application.yml +++ b/metron-interface/metron-rest/src/main/resources/application.yml @@ -72,6 +72,7 @@ user: settings: table: user_settings cf: cf + job.limit: 1 pcap: base.path: /apps/metron/pcap/input http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java index 25956e4..5d30e72 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.metron.common.Constants; import org.apache.metron.job.JobStatus; import org.apache.metron.job.Pageable; +import org.apache.metron.job.manager.InMemoryJobManager; import org.apache.metron.pcap.PcapHelper; import org.apache.metron.pcap.PcapPages; import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; @@ -127,6 +128,8 @@ public class PcapControllerIntegrationTest { @Before public void setup() throws Exception { this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + InMemoryJobManager jobManager = (InMemoryJobManager) wac.getBean("jobManager"); + jobManager.clear(); } @Test @@ -216,6 +219,25 @@ public class PcapControllerIntegrationTest { } @Test + public void testTooManyJobs() throws Exception { + MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); + + mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.RUNNING)); + + this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.jobId").value("jobId")) + .andExpect(jsonPath("$.jobStatus").value("RUNNING")); + + this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) + .andExpect(status().isInternalServerError()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.message").value("Cannot submit job because a job is already running. Please contact the administrator to cancel job(s) with id(s) jobId")); + + } + + @Test public void testGetStatus() throws Exception { MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); @@ -265,6 +287,30 @@ public class PcapControllerIntegrationTest { } @Test + public void testGetStatusList() throws Exception { + MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); + + mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.RUNNING)); + + this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.jobId").value("jobId")) + .andExpect(jsonPath("$.jobStatus").value("RUNNING")); + + this.mockMvc.perform(get(pcapUrl + "?state=RUNNING").with(httpBasic(user, password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[0].jobId").value("jobId")) + .andExpect(jsonPath("$[0].jobStatus").value("RUNNING")); + + this.mockMvc.perform(get(pcapUrl + "?state=SUCCEEDED").with(httpBasic(user, password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(content().json("[]")); + } + + @Test public void testKillJob() throws Exception { MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java index e9f8f9f..f99ab93 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java @@ -28,6 +28,7 @@ import org.apache.metron.job.JobException; import org.apache.metron.job.JobNotFoundException; import org.apache.metron.job.JobStatus; import org.apache.metron.job.Pageable; +import org.apache.metron.job.Statusable; import org.apache.metron.job.manager.InMemoryJobManager; import org.apache.metron.job.manager.JobManager; import org.apache.metron.pcap.PcapHelper; @@ -58,6 +59,8 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -189,6 +192,7 @@ public class PcapServiceImplTest { when(environment.getProperty(MetronRestConstants.PCAP_FINAL_OUTPUT_PATH_SPRING_PROPERTY)).thenReturn("/final/output/path"); when(environment.getProperty(MetronRestConstants.PCAP_PAGE_SIZE_SPRING_PROPERTY)).thenReturn("100"); when(environment.getProperty(MetronRestConstants.PCAP_PDML_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/path/to/pdml/script"); + when(environment.getProperty(MetronRestConstants.USER_JOB_LIMIT_SPRING_PROPERTY, Integer.class, 1)).thenReturn(1); } @Test @@ -334,6 +338,26 @@ public class PcapServiceImplTest { } @Test + public void submitShouldThrowExceptionOnRunningJobFound() throws Exception { + exception.expect(RestException.class); + exception.expectMessage("Cannot submit job because a job is already running. Please contact the administrator to cancel job(s) with id(s) jobId"); + + PcapStatus runningStatus1 = new PcapStatus(); + runningStatus1.setJobStatus("RUNNING"); + runningStatus1.setJobId("jobId1"); + PcapStatus runningStatus2 = new PcapStatus(); + runningStatus2.setJobStatus("RUNNING"); + runningStatus2.setJobId("jobId2"); + + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, new InMemoryJobManager<>(), pcapToPdmlScriptWrapper)); + doReturn(Arrays.asList(runningStatus1, runningStatus2)).when(pcapService).getJobStatus("user", JobStatus.State.RUNNING); + when(environment.getProperty(MetronRestConstants.USER_JOB_LIMIT_SPRING_PROPERTY, Integer.class, 1)).thenReturn(2); + + pcapService.submit("user", new FixedPcapRequest()); + } + + + @Test public void fixedShouldThrowRestException() throws Exception { exception.expect(RestException.class); exception.expectMessage("some job exception"); @@ -397,6 +421,50 @@ public class PcapServiceImplTest { } @Test + public void getStatusForStateShouldProperlyReturnJobs() throws Exception { + MockPcapJob mockPcapJob = mock(MockPcapJob.class); + JobManager jobManager = mock(JobManager.class); + Statusable<Path> runningJob = mock(Statusable.class); + JobStatus runningStatus = mock(JobStatus.class); + when(runningStatus.getJobId()).thenReturn("runningJob"); + when(runningStatus.getState()).thenReturn(JobStatus.State.RUNNING); + when(runningJob.getStatus()).thenReturn(runningStatus); + + Statusable<Path> failedJob = mock(Statusable.class); + when(failedJob.getStatus()).thenThrow(new JobException("job exception")); + + Statusable<Path> succeededJob = mock(Statusable.class); + JobStatus succeededStatus = mock(JobStatus.class); + when(succeededStatus.getJobId()).thenReturn("succeededJob"); + when(succeededStatus.getState()).thenReturn(JobStatus.State.SUCCEEDED); + when(succeededJob.isDone()).thenReturn(true); + when(succeededJob.getStatus()).thenReturn(succeededStatus); + Pageable<Path> succeededPageable = mock(Pageable.class); + when(succeededPageable.getSize()).thenReturn(5); + when(succeededJob.get()).thenReturn(succeededPageable); + + when(jobManager.getJobs("user")).thenReturn(Arrays.asList(runningJob, failedJob, succeededJob)); + + PcapServiceImpl pcapService = new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper); + + PcapStatus expectedRunningPcapStatus = new PcapStatus(); + expectedRunningPcapStatus.setJobId("runningJob"); + expectedRunningPcapStatus.setJobStatus(JobStatus.State.RUNNING.name()); + Assert.assertEquals(expectedRunningPcapStatus, pcapService.getJobStatus("user", JobStatus.State.RUNNING).get(0)); + + PcapStatus expectedFailedPcapStatus = new PcapStatus(); + expectedFailedPcapStatus.setJobStatus(JobStatus.State.FAILED.name()); + expectedFailedPcapStatus.setDescription("job exception"); + Assert.assertEquals(expectedFailedPcapStatus, pcapService.getJobStatus("user", JobStatus.State.FAILED).get(0)); + + PcapStatus expectedSucceededPcapStatus = new PcapStatus(); + expectedSucceededPcapStatus.setJobId("succeededJob"); + expectedSucceededPcapStatus.setJobStatus(JobStatus.State.SUCCEEDED.name()); + expectedSucceededPcapStatus.setPageTotal(5); + Assert.assertEquals(expectedSucceededPcapStatus, pcapService.getJobStatus("user", JobStatus.State.SUCCEEDED).get(0)); + } + + @Test public void killJobShouldKillJobAndReportStatus() throws Exception { MockPcapJob mockPcapJob = mock(MockPcapJob.class); JobManager jobManager = mock(JobManager.class); http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java b/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java index 807af4d..ba436b3 100644 --- a/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java +++ b/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java @@ -87,4 +87,8 @@ public class InMemoryJobManager<PAGE_T> implements JobManager<PAGE_T> { return new ArrayList<>(getUserJobs(username).values()); } + public void clear() { + jobs.clear(); + } + }