METRON-503: Metron REST API this closes apache/incubator-metron#316
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/e6628499 Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/e6628499 Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/e6628499 Branch: refs/heads/master Commit: e662849932a74187708b0037042c6b06d4750116 Parents: a61dbcf Author: merrimanr <merrim...@gmail.com> Authored: Thu Mar 2 11:04:06 2017 -0500 Committer: cstella <ceste...@gmail.com> Committed: Thu Mar 2 15:24:11 2017 -0500 ---------------------------------------------------------------------- dependencies_with_url.csv | 3 + metron-interface/metron-rest-client/pom.xml | 2 +- .../metron/rest/model/GrokValidation.java | 13 +- .../metron/rest/model/ParseMessageRequest.java | 9 + .../metron/rest/model/SensorParserContext.java | 51 +++ .../metron/rest/model/TopologyResponse.java | 18 + .../metron/rest/model/TopologyStatus.java | 28 +- .../metron/rest/model/TopologySummary.java | 17 + .../rest/model/TransformationValidation.java | 51 --- metron-interface/metron-rest/.gitignore | 1 + metron-interface/metron-rest/README.md | 222 ++++++++---- metron-interface/metron-rest/pom.xml | 39 +-- .../metron/rest/controller/HdfsController.java | 89 +++++ .../rest/controller/StellarController.java | 80 +++++ .../controller/TransformationController.java | 80 ----- .../apache/metron/rest/service/GrokService.java | 3 + .../apache/metron/rest/service/HdfsService.java | 13 +- .../rest/service/SensorParserConfigService.java | 3 + .../metron/rest/service/StellarService.java | 39 +++ .../rest/service/TransformationService.java | 39 --- .../rest/service/impl/GrokServiceImpl.java | 59 +++- .../rest/service/impl/HdfsServiceImpl.java | 55 ++- .../impl/SensorEnrichmentConfigServiceImpl.java | 8 +- .../impl/SensorIndexingConfigServiceImpl.java | 8 +- .../impl/SensorParserConfigServiceImpl.java | 141 +------- .../rest/service/impl/StellarServiceImpl.java | 92 +++++ .../service/impl/StormAdminServiceImpl.java | 13 +- .../rest/service/impl/StormCLIWrapper.java | 6 +- .../service/impl/StormStatusServiceImpl.java | 8 +- .../service/impl/TransformationServiceImpl.java | 92 ----- .../src/main/resources/application-docker.yml | 6 +- .../src/main/resources/application-vagrant.yml | 51 +++ .../src/main/resources/application.yml | 2 +- .../metron-rest/src/main/scripts/start.sh | 33 -- .../src/main/scripts/start_metron_rest.sh | 25 ++ .../GlobalConfigControllerIntegrationTest.java | 8 + .../GrokControllerIntegrationTest.java | 24 +- .../HdfsControllerIntegrationTest.java | 101 ++++++ ...richmentConfigControllerIntegrationTest.java | 23 +- ...IndexingConfigControllerIntegrationTest.java | 2 + ...orParserConfigControllerIntegrationTest.java | 36 +- .../StellarControllerIntegrationTest.java | 122 +++++++ .../StormControllerIntegrationTest.java | 3 + ...TransformationControllerIntegrationTest.java | 122 ------- .../rest/service/SensorParserConfigTest.java | 116 ------- .../service/impl/DockerStormCLIWrapperTest.java | 14 +- .../rest/service/impl/GrokServiceImplTest.java | 136 ++++++-- .../impl/HdfsServiceImplExceptionTest.java | 101 ++++++ .../rest/service/impl/HdfsServiceImplTest.java | 143 ++++---- .../rest/service/impl/KafkaServiceImplTest.java | 4 +- .../SensorEnrichmentConfigServiceImplTest.java | 256 ++++++++++++++ .../SensorIndexingConfigServiceImplTest.java | 234 +++++++++++++ .../impl/SensorParserConfigServiceImplTest.java | 344 +++++++++++++++++++ .../service/impl/StellarServiceImplTest.java | 92 +++++ .../service/impl/StormAdminServiceImplTest.java | 156 +++++++++ .../rest/service/impl/StormCLIWrapperTest.java | 216 ++++++++++++ .../impl/StormStatusServiceImplTest.java | 221 ++++++++++++ .../metron-rest/src/test/resources/README.vm | 121 +++++-- metron-interface/pom.xml | 2 +- 59 files changed, 3013 insertions(+), 983 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/dependencies_with_url.csv ---------------------------------------------------------------------- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 35acced..1d428c3 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -283,3 +283,6 @@ org.springframework.security:spring-security-config:jar:4.1.3.RELEASE:compile,AS org.springframework.security:spring-security-core:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security org.springframework.security:spring-security-web:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security antlr:antlr:jar:2.7.7:compile,BSD 3-Clause License,http://www.antlr2.org +com.h2database:h2:jar:1.4.192:compile,EPL 1.0,http://www.h2database.com/html/license.html +de.jollyday:jollyday:jar:0.5.2:compile,ASLv2,http://jollyday.sourceforge.net/license.html +org.threeten:threeten-extra:jar:1.0:compile,BSD,http://www.threeten.org/threeten-extra/license.html http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml index 66ccf52..0a49986 100644 --- a/metron-interface/metron-rest-client/pom.xml +++ b/metron-interface/metron-rest-client/pom.xml @@ -18,7 +18,7 @@ <parent> <groupId>org.apache.metron</groupId> <artifactId>metron-interface</artifactId> - <version>0.3.0</version> + <version>0.3.1</version> </parent> <artifactId>metron-rest-client</artifactId> <properties> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java index ccd2c5c..2795320 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java @@ -22,10 +22,19 @@ import java.util.Map; public class GrokValidation { + private String patternLabel; private String statement; private String sampleData; private Map<String, Object> results; + public String getPatternLabel() { + return patternLabel; + } + + public void setPatternLabel(String patternLabel) { + this.patternLabel = patternLabel; + } + public String getStatement() { return statement; } @@ -60,6 +69,7 @@ public class GrokValidation { GrokValidation that = (GrokValidation) o; + if (patternLabel != null ? !patternLabel.equals(that.patternLabel) : that.patternLabel != null) return false; if (statement != null ? !statement.equals(that.statement) : that.statement != null) return false; if (sampleData != null ? !sampleData.equals(that.sampleData) : that.sampleData != null) return false; return results != null ? results.equals(that.results) : that.results == null; @@ -67,7 +77,8 @@ public class GrokValidation { @Override public int hashCode() { - int result = statement != null ? statement.hashCode() : 0; + int result = patternLabel != null ? patternLabel.hashCode() : 0; + result = 31 * result + (statement != null ? statement.hashCode() : 0); result = 31 * result + (sampleData != null ? sampleData.hashCode() : 0); result = 31 * result + (results != null ? results.hashCode() : 0); return result; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java index 8dfe17b..50eb88d 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java @@ -22,6 +22,7 @@ import org.apache.metron.common.configuration.SensorParserConfig; public class ParseMessageRequest { private SensorParserConfig sensorParserConfig; + private String grokStatement; private String sampleData; public SensorParserConfig getSensorParserConfig() { @@ -32,6 +33,14 @@ public class ParseMessageRequest { this.sensorParserConfig = sensorParserConfig; } + public String getGrokStatement() { + return grokStatement; + } + + public void setGrokStatement(String grokStatement) { + this.grokStatement = grokStatement; + } + public String getSampleData() { return sampleData; } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java new file mode 100644 index 0000000..cb10cfe --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import org.apache.metron.common.configuration.SensorParserConfig; + +import java.util.HashMap; +import java.util.Map; + +public class SensorParserContext { + + private Map<String, Object> sampleData; + private SensorParserConfig sensorParserConfig; + + public Map<String, Object> getSampleData() { + if (sampleData == null) { + return new HashMap<>(); + } + return sampleData; + } + + public void setSampleData(Map<String, Object> sampleData) { + this.sampleData = sampleData; + } + + public SensorParserConfig getSensorParserConfig() { + if (sensorParserConfig == null) { + return new SensorParserConfig(); + } + return sensorParserConfig; + } + + public void setSensorParserConfig(SensorParserConfig sensorParserConfig) { + this.sensorParserConfig = sensorParserConfig; + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java index 6894e89..0ad2e9f 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java @@ -39,4 +39,22 @@ public class TopologyResponse { this.status = TopologyResponseCode.ERROR; this.message = message; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TopologyResponse that = (TopologyResponse) o; + + if (status != null ? !status.equals(that.status) : that.status != null) return false; + return message != null ? message.equals(that.message) : that.message == null; + } + + @Override + public int hashCode() { + int result = status != null ? status.hashCode() : 0; + result = 31 * result + (message != null ? message.hashCode() : 0); + return result; + } } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java index 0e2d28a..6fc1c01 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java @@ -26,8 +26,8 @@ public class TopologyStatus { private String name; private TopologyStatusCode status; private Map<String, Object>[] topologyStats; - private double latency = 0; - private double throughput = 0; + private Double latency = 0.0; + private Double throughput = 0.0; public String getId() { return id; @@ -73,4 +73,28 @@ public class TopologyStatus { } } } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TopologyStatus that = (TopologyStatus) o; + + if (id != null ? !id.equals(that.id) : that.id != null) return false; + if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (status != null ? !status.equals(that.status) : that.status != null) return false; + if (!latency.equals(that.latency)) return false; + return throughput.equals(that.throughput); + } + + @Override + public int hashCode() { + int result = id != null ? id.hashCode() : 0; + result = 31 * result + (name != null ? name.hashCode() : 0); + result = 31 * result + (status != null ? status.hashCode() : 0); + result = 31 * result + (latency != null ? latency.hashCode() : 0); + result = 31 * result + (throughput != null ? throughput.hashCode() : 0); + return result; + } } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java index b9d39a4..8621daf 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java @@ -17,6 +17,8 @@ */ package org.apache.metron.rest.model; +import java.util.Arrays; + public class TopologySummary { private TopologyStatus[] topologies; @@ -31,4 +33,19 @@ public class TopologySummary { public void setTopologies(TopologyStatus[] topologies) { this.topologies = topologies; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TopologySummary that = (TopologySummary) o; + + return topologies != null ? Arrays.equals(topologies, that.topologies) : that.topologies != null; + } + + @Override + public int hashCode() { + return topologies != null ? Arrays.hashCode(topologies) : 0; + } } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java deleted file mode 100644 index c2e39e4..0000000 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.rest.model; - -import org.apache.metron.common.configuration.SensorParserConfig; - -import java.util.HashMap; -import java.util.Map; - -public class TransformationValidation { - - private Map<String, Object> sampleData; - private SensorParserConfig sensorParserConfig; - - public Map<String, Object> getSampleData() { - if (sampleData == null) { - return new HashMap<>(); - } - return sampleData; - } - - public void setSampleData(Map<String, Object> sampleData) { - this.sampleData = sampleData; - } - - public SensorParserConfig getSensorParserConfig() { - if (sensorParserConfig == null) { - return new SensorParserConfig(); - } - return sensorParserConfig; - } - - public void setSensorParserConfig(SensorParserConfig sensorParserConfig) { - this.sensorParserConfig = sensorParserConfig; - } -} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/.gitignore ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/.gitignore b/metron-interface/metron-rest/.gitignore new file mode 100644 index 0000000..26b3af1 --- /dev/null +++ b/metron-interface/metron-rest/.gitignore @@ -0,0 +1 @@ +metrondb.*.db http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 63a2d34..cf41bf3 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -1,66 +1,83 @@ -# Metron REST and Configuration UI +# Metron REST -This UI exposes and aids in sensor configuration. +This module provides a RESTful API for interacting with Metron. ## Prerequisites * A running Metron cluster -* A running instance of MySQL * Java 8 installed * Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed ## Installation -1. Package the Application with Maven: - ``` - mvn clean package - ``` +1. Package the application with Maven: +``` +mvn clean package +``` 1. Untar the archive in the target directory. The directory structure will look like: - ``` - bin - start.sh - lib - metron-rest-version.jar - ``` +``` +bin + start_metron_rest.sh +lib + metron-rest-$METRON_VERSION.jar +``` + +1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details). -1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/). Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory. - ``` - export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final - ``` +1. Start the application with this command: +``` +./bin/start_metron_rest.sh /path/to/application.yml +``` -1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/). Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory. - ``` - export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40 - ``` +## Usage -1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html). +The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. -1. Create a Config UI database in MySQL with this command: - ``` - CREATE DATABASE IF NOT EXISTS metronrest - ``` +## Security -1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4. +The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users: +``` +./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev +``` -1. Start the UI with this command: - ``` - ./bin/start.sh /path/to/application.yml - ``` +For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows: -## Usage +1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html). -The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. Users can be added with this SQL statement: +1. Connect to MySQL and create a Metron REST database: +``` +CREATE DATABASE IF NOT EXISTS metronrest +``` + +1. Add users: ``` use metronrest; insert into users (username, password, enabled) values ('your_username','your_password',1); insert into authorities (username, authority) values ('your_username', 'ROLE_USER'); ``` -Users can be added to additional groups with this SQL statement: + +1. Replace the H2 connection information in the application.yml file with MySQL connection information: ``` -use metronrest; -insert into authorities (username, authority) values ('your_username', 'your_group'); +spring: + datasource: + driverClassName: com.mysql.jdbc.Driver + url: jdbc:mysql://mysql_host:3306/metronrest + username: metron_rest_user + password: metron_rest_password + platform: mysql ``` +1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml: +``` +<dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + <version>${mysql.client.version}</version> +</dependency> +``` + +1. Follow the steps in the [Installation](#installation) section + ## API Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI. @@ -72,6 +89,10 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `POST /api/v1/global/config`](#post-apiv1globalconfig)| | [ `GET /api/v1/grok/list`](#get-apiv1groklist)| | [ `POST /api/v1/grok/validate`](#post-apiv1grokvalidate)| +| [ `POST /api/v1/hdfs`](#post-apiv1hdfs)| +| [ `GET /api/v1/hdfs`](#get-apiv1hdfs)| +| [ `DELETE /api/v1/hdfs`](#delete-apiv1hdfs)| +| [ `GET /api/v1/hdfs/list`](#get-apiv1hdfslist)| | [ `GET /api/v1/kafka/topic`](#get-apiv1kafkatopic)| | [ `POST /api/v1/kafka/topic`](#post-apiv1kafkatopic)| | [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopic{name})| @@ -93,6 +114,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/sensor/parser/config/reload/available`](#get-apiv1sensorparserconfigreloadavailable)| | [ `DELETE /api/v1/sensor/parser/config/{name}`](#delete-apiv1sensorparserconfig{name})| | [ `GET /api/v1/sensor/parser/config/{name}`](#get-apiv1sensorparserconfig{name})| +| [ `POST /api/v1/stellar/apply/transformations`](#post-apiv1stellarapplytransformations)| +| [ `GET /api/v1/stellar/list`](#get-apiv1stellarlist)| +| [ `GET /api/v1/stellar/list/functions`](#get-apiv1stellarlistfunctions)| +| [ `GET /api/v1/stellar/list/simple/functions`](#get-apiv1stellarlistsimplefunctions)| +| [ `POST /api/v1/stellar/validate/rules`](#post-apiv1stellarvalidaterules)| | [ `GET /api/v1/storm`](#get-apiv1storm)| | [ `GET /api/v1/storm/client/status`](#get-apiv1stormclientstatus)| | [ `GET /api/v1/storm/enrichment`](#get-apiv1stormenrichment)| @@ -110,11 +136,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/storm/parser/start/{name}`](#get-apiv1stormparserstart{name})| | [ `GET /api/v1/storm/parser/stop/{name}`](#get-apiv1stormparserstop{name})| | [ `GET /api/v1/storm/{name}`](#get-apiv1storm{name})| -| [ `GET /api/v1/transformation/list`](#get-apiv1transformationlist)| -| [ `GET /api/v1/transformation/list/functions`](#get-apiv1transformationlistfunctions)| -| [ `GET /api/v1/transformation/list/simple/functions`](#get-apiv1transformationlistsimplefunctions)| -| [ `POST /api/v1/transformation/validate`](#post-apiv1transformationvalidate)| -| [ `POST /api/v1/transformation/validate/rules`](#post-apiv1transformationvalidaterules)| | [ `GET /api/v1/user`](#get-apiv1user)| ### `GET /api/v1/global/config` @@ -145,10 +166,41 @@ Request and Response objects are JSON formatted. The JSON schemas are available ### `POST /api/v1/grok/validate` * Description: Applies a Grok statement to a sample message * Input: - * grokValidation - Object containing Grok statment and sample message + * grokValidation - Object containing Grok statement and sample message * Returns: * 200 - JSON results +### `POST /api/v1/hdfs` + * Description: Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists. + * Input: + * path - Path to HDFS file + * contents - File contents + * Returns: + * 200 - Contents were written + +### `GET /api/v1/hdfs` + * Description: Reads a file from HDFS and returns the contents + * Input: + * path - Path to HDFS file + * Returns: + * 200 - Returns file contents + +### `DELETE /api/v1/hdfs` + * Description: Deletes a file from HDFS + * Input: + * path - Path to HDFS file + * recursive - Delete files recursively + * Returns: + * 200 - File was deleted + * 404 - File was not found in HDFS + +### `GET /api/v1/hdfs/list` + * Description: Reads a file from HDFS and returns the contents + * Input: + * path - Path to HDFS directory + * Returns: + * 200 - Returns file contents + ### `GET /api/v1/kafka/topic` * Description: Retrieves all Kafka topics * Returns: @@ -296,6 +348,35 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns SensorParserConfig * 404 - SensorParserConfig is missing +### `POST /api/v1/stellar/apply/transformations` + * Description: Executes transformations against a sample message + * Input: + * transformationValidation - Object containing SensorParserConfig and sample message + * Returns: + * 200 - Returns transformation results + +### `GET /api/v1/stellar/list` + * Description: Retrieves field transformations + * Returns: + * 200 - Returns a list field transformations + +### `GET /api/v1/stellar/list/functions` + * Description: Lists the Stellar functions that can be found on the classpath + * Returns: + * 200 - Returns a list of Stellar functions + +### `GET /api/v1/stellar/list/simple/functions` + * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath + * Returns: + * 200 - Returns a list of simple Stellar functions + +### `POST /api/v1/stellar/validate/rules` + * Description: Tests Stellar statements to ensure they are well-formed + * Input: + * statements - List of statements to validate + * Returns: + * 200 - Returns validation results + ### `GET /api/v1/storm` * Description: Retrieves the status of all Storm topologies * Returns: @@ -399,40 +480,43 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns topology status information * 404 - Topology is missing -### `GET /api/v1/transformation/list` - * Description: Retrieves field transformations +### `GET /api/v1/user` + * Description: Retrieves the current user * Returns: - * 200 - Returns a list field transformations + * 200 - Current user -### `GET /api/v1/transformation/list/functions` - * Description: Lists the Stellar functions that can be found on the classpath - * Returns: - * 200 - Returns a list of Stellar functions +## Testing -### `GET /api/v1/transformation/list/simple/functions` - * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath - * Returns: - * 200 - Returns a list of simple Stellar functions +Profiles are includes for both the metron-docker and Quick Dev environments. -### `POST /api/v1/transformation/validate` - * Description: Executes transformations against a sample message - * Input: - * transformationValidation - Object containing SensorParserConfig and sample message - * Returns: - * 200 - Returns transformation results +### metron-docker -### `POST /api/v1/transformation/validate/rules` - * Description: Tests Stellar statements to ensure they are well-formed - * Input: - * statements - List of statements to validate - * Returns: - * 200 - Returns validation results +Start the [metron-docker](../../metron-docker) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin: +``` +mvn clean package +mvn spring-boot:run -Drun.profiles=docker,dev +``` +The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/. -### `GET /api/v1/user` - * Description: Retrieves the current user - * Returns: - * 200 - Current user +### Quick Dev +Start the [Quick Dev](../../metron-deployment/vagrant/quick-dev-platform) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin: +``` +mvn clean package +mvn spring-boot:run -Drun.profiles=vagrant,dev +``` +The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/. + +To run the application locally on the Quick Dev host, package the application and scp the archive to node1: +``` +mvn clean package +scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:~/ +``` +Login to node1 and unarchive the metron-rest application. Start the application on a different port to avoid conflicting with Ambari: +``` +java -jar ./lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082 +``` +The metron-rest application will be available at http://node1:8082/swagger-ui.html#/. ## License http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index c2beb2d..849455e 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -18,7 +18,7 @@ <parent> <groupId>org.apache.metron</groupId> <artifactId>metron-interface</artifactId> - <version>0.3.0</version> + <version>0.3.1</version> </parent> <artifactId>metron-rest</artifactId> <properties> @@ -61,15 +61,15 @@ <artifactId>spring-boot-starter-jdbc</artifactId> </dependency> <dependency> - <groupId>mysql</groupId> - <artifactId>mysql-connector-java</artifactId> - <version>${mysql.client.version}</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> <version>${curator.version}</version> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.googlecode.json-simple</groupId> @@ -105,6 +105,10 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </exclusion> + <exclusion> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -122,8 +126,16 @@ </exclusion> <exclusion> <groupId>org.apache.metron</groupId> + <artifactId>metron-statistics</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.metron</groupId> <artifactId>metron-writer</artifactId> </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -147,18 +159,6 @@ <version>0.1.0</version> </dependency> <dependency> - <groupId>org.reflections</groupId> - <artifactId>reflections</artifactId> - <version>0.9.10</version> - <exclusions> - <exclusion> - <groupId>com.google.code.findbugs</groupId> - <artifactId>annotations</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> @@ -166,7 +166,6 @@ <dependency> <groupId>com.h2database</groupId> <artifactId>h2</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.springframework.security</groupId> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java new file mode 100644 index 0000000..bf7ae84 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.HdfsService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; + +@RestController +@RequestMapping("/api/v1/hdfs") +public class HdfsController { + + @Autowired + private HdfsService hdfsService; + + @ApiOperation(value = "Reads a file from HDFS and returns the contents") + @ApiResponse(message = "Returns file contents", code = 200) + @RequestMapping(value = "/list", method = RequestMethod.GET) + ResponseEntity<List<String>> list(@ApiParam(name = "path", value = "Path to HDFS directory", required = true) @RequestParam String path) throws RestException { + return new ResponseEntity<>(hdfsService.list(new Path(path)), HttpStatus.OK); + } + + @ApiOperation(value = "Reads a file from HDFS and returns the contents") + @ApiResponse(message = "Returns file contents", code = 200) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity<String> read(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path) throws RestException { + String contents = hdfsService.read(new Path(path)); + if (contents != null) { + return new ResponseEntity<>(hdfsService.read(new Path(path)), HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + + } + + @ApiOperation(value = "Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists.") + @ApiResponse(message = "Contents were written", code = 200) + @RequestMapping(method = RequestMethod.POST) + ResponseEntity<Void> write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path, + @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException { + hdfsService.write(new Path(path), contents.getBytes(UTF_8)); + return new ResponseEntity<>(HttpStatus.OK); + + } + + @ApiOperation(value = "Deletes a file from HDFS") + @ApiResponses(value = { @ApiResponse(message = "File was deleted", code = 200), + @ApiResponse(message = "File was not found in HDFS", code = 404) }) + @RequestMapping(method = RequestMethod.DELETE) + ResponseEntity<Boolean> delete(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path, + @ApiParam(name = "recursive", value = "Delete files recursively") @RequestParam(required = false, defaultValue = "false") boolean recursive) throws RestException { + if (hdfsService.delete(new Path(path), recursive)) { + return new ResponseEntity<>(HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java new file mode 100644 index 0000000..13327f9 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import org.apache.metron.common.field.transformation.FieldTransformations; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.StellarFunctionDescription; +import org.apache.metron.rest.model.SensorParserContext; +import org.apache.metron.rest.service.StellarService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/stellar") +public class StellarController { + + @Autowired + private StellarService stellarService; + + @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed") + @ApiResponse(message = "Returns validation results", code = 200) + @RequestMapping(value = "/validate/rules", method = RequestMethod.POST) + ResponseEntity<Map<String, Boolean>> validateRules(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException { + return new ResponseEntity<>(stellarService.validateRules(statements), HttpStatus.OK); + } + + @ApiOperation(value = "Executes transformations against a sample message") + @ApiResponse(message = "Returns transformation results", code = 200) + @RequestMapping(value = "/apply/transformations", method = RequestMethod.POST) + ResponseEntity<Map<String, Object>> applyTransformations(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody SensorParserContext sensorParserContext) throws RestException { + return new ResponseEntity<>(stellarService.applyTransformations(sensorParserContext), HttpStatus.OK); + } + + @ApiOperation(value = "Retrieves field transformations") + @ApiResponse(message = "Returns a list field transformations", code = 200) + @RequestMapping(value = "/list", method = RequestMethod.GET) + ResponseEntity<FieldTransformations[]> list() throws RestException { + return new ResponseEntity<>(stellarService.getTransformations(), HttpStatus.OK); + } + + @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath") + @ApiResponse(message = "Returns a list of Stellar functions", code = 200) + @RequestMapping(value = "/list/functions", method = RequestMethod.GET) + ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException { + return new ResponseEntity<>(stellarService.getStellarFunctions(), HttpStatus.OK); + } + + @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath") + @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200) + @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET) + ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException { + return new ResponseEntity<>(stellarService.getSimpleStellarFunctions(), HttpStatus.OK); + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java deleted file mode 100644 index bc8b201..0000000 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.rest.controller; - -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; -import io.swagger.annotations.ApiResponse; -import org.apache.metron.common.field.transformation.FieldTransformations; -import org.apache.metron.rest.RestException; -import org.apache.metron.rest.model.StellarFunctionDescription; -import org.apache.metron.rest.model.TransformationValidation; -import org.apache.metron.rest.service.TransformationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; - -import java.util.List; -import java.util.Map; - -@RestController -@RequestMapping("/api/v1/transformation") -public class TransformationController { - - @Autowired - private TransformationService transformationService; - - @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed") - @ApiResponse(message = "Returns validation results", code = 200) - @RequestMapping(value = "/validate/rules", method = RequestMethod.POST) - ResponseEntity<Map<String, Boolean>> validateRule(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException { - return new ResponseEntity<>(transformationService.validateRules(statements), HttpStatus.OK); - } - - @ApiOperation(value = "Executes transformations against a sample message") - @ApiResponse(message = "Returns transformation results", code = 200) - @RequestMapping(value = "/validate", method = RequestMethod.POST) - ResponseEntity<Map<String, Object>> validateTransformation(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody TransformationValidation transformationValidation) throws RestException { - return new ResponseEntity<>(transformationService.validateTransformation(transformationValidation), HttpStatus.OK); - } - - @ApiOperation(value = "Retrieves field transformations") - @ApiResponse(message = "Returns a list field transformations", code = 200) - @RequestMapping(value = "/list", method = RequestMethod.GET) - ResponseEntity<FieldTransformations[]> list() throws RestException { - return new ResponseEntity<>(transformationService.getTransformations(), HttpStatus.OK); - } - - @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath") - @ApiResponse(message = "Returns a list of Stellar functions", code = 200) - @RequestMapping(value = "/list/functions", method = RequestMethod.GET) - ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException { - return new ResponseEntity<>(transformationService.getStellarFunctions(), HttpStatus.OK); - } - - @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath") - @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200) - @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET) - ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException { - return new ResponseEntity<>(transformationService.getSimpleStellarFunctions(), HttpStatus.OK); - } -} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java index 95ce1ba..268d396 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java @@ -20,6 +20,7 @@ package org.apache.metron.rest.service; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.GrokValidation; +import java.io.File; import java.util.Map; public interface GrokService { @@ -28,4 +29,6 @@ public interface GrokService { GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException; + File saveTemporary(String statement, String name) throws RestException; + } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java index 97888ff..d5932c7 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java @@ -17,19 +17,18 @@ */ package org.apache.metron.rest.service; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.springframework.stereotype.Service; +import org.apache.metron.rest.RestException; -import java.io.IOException; +import java.util.List; public interface HdfsService { - byte[] read(Path path) throws IOException; + String read(Path path) throws RestException; - void write(Path path, byte[] contents) throws IOException; + void write(Path path, byte[] contents) throws RestException; - FileStatus[] list(Path path) throws IOException; + List<String> list(Path path) throws RestException; - boolean delete(Path path, boolean recursive) throws IOException; + boolean delete(Path path, boolean recursive) throws RestException; } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java index efda639..9b863b8 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java @@ -22,6 +22,7 @@ import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.ParseMessageRequest; import org.json.simple.JSONObject; +import java.util.List; import java.util.Map; public interface SensorParserConfigService { @@ -32,6 +33,8 @@ public interface SensorParserConfigService { Iterable<SensorParserConfig> getAll() throws RestException; + List<String> getAllTypes() throws RestException; + boolean delete(String name) throws RestException; Map<String, String> getAvailableParsers(); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java new file mode 100644 index 0000000..14cd31f --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.common.field.transformation.FieldTransformations; +import org.apache.metron.rest.model.StellarFunctionDescription; +import org.apache.metron.rest.model.SensorParserContext; + +import java.util.List; +import java.util.Map; + +public interface StellarService { + + Map<String, Boolean> validateRules(List<String> rules); + + Map<String, Object> applyTransformations(SensorParserContext sensorParserContext); + + FieldTransformations[] getTransformations(); + + List<StellarFunctionDescription> getStellarFunctions(); + + List<StellarFunctionDescription> getSimpleStellarFunctions(); + +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java deleted file mode 100644 index d1400c6..0000000 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.rest.service; - -import org.apache.metron.common.field.transformation.FieldTransformations; -import org.apache.metron.rest.model.StellarFunctionDescription; -import org.apache.metron.rest.model.TransformationValidation; - -import java.util.List; -import java.util.Map; - -public interface TransformationService { - - Map<String, Boolean> validateRules(List<String> rules); - - Map<String, Object> validateTransformation(TransformationValidation transformationValidation); - - FieldTransformations[] getTransformations(); - - List<StellarFunctionDescription> getStellarFunctions(); - - List<StellarFunctionDescription> getSimpleStellarFunctions(); - -} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java index 323ca78..3f2de2f 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java @@ -20,22 +20,35 @@ package org.apache.metron.rest.service.impl; import oi.thekraken.grok.api.Grok; import oi.thekraken.grok.api.Match; import org.apache.directory.api.util.Strings; +import org.apache.hadoop.fs.Path; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.GrokValidation; import org.apache.metron.rest.service.GrokService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.io.InputStreamReader; import java.io.StringReader; import java.util.Map; +import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; + @Service public class GrokServiceImpl implements GrokService { + + private Environment environment; + private Grok commonGrok; @Autowired - public GrokServiceImpl(Grok commonGrok) { + public GrokServiceImpl(Environment environment, Grok commonGrok) { + this.environment = environment; this.commonGrok = commonGrok; } @@ -48,20 +61,21 @@ public class GrokServiceImpl implements GrokService { public GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException { Map<String, Object> results; try { - String statement = Strings.isEmpty(grokValidation.getStatement()) ? "" : grokValidation.getStatement(); - + if (grokValidation.getPatternLabel() == null) { + throw new RestException("Pattern label is required"); + } + if (Strings.isEmpty(grokValidation.getStatement())) { + throw new RestException("Grok statement is required"); + } Grok grok = new Grok(); grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common"))); - grok.addPatternFromReader(new StringReader(statement)); - String patternLabel = statement.substring(0, statement.indexOf(" ")); - String grokPattern = "%{" + patternLabel + "}"; + grok.addPatternFromReader(new StringReader(grokValidation.getStatement())); + String grokPattern = "%{" + grokValidation.getPatternLabel() + "}"; grok.compile(grokPattern); Match gm = grok.match(grokValidation.getSampleData()); gm.captures(); results = gm.toMap(); - results.remove(patternLabel); - } catch (StringIndexOutOfBoundsException e) { - throw new RestException("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)", e.getCause()); + results.remove(grokValidation.getPatternLabel()); } catch (Exception e) { throw new RestException(e); } @@ -69,4 +83,31 @@ public class GrokServiceImpl implements GrokService { return grokValidation; } + @Override + public File saveTemporary(String statement, String name) throws RestException { + if (statement != null) { + try { + File grokDirectory = new File(getTemporaryGrokRootPath()); + if (!grokDirectory.exists()) { + grokDirectory.mkdirs(); + } + File path = new File(grokDirectory, name); + FileWriter fileWriter = new FileWriter(new File(grokDirectory, name)); + fileWriter.write(statement); + fileWriter.close(); + return path; + } catch (IOException e) { + throw new RestException(e); + } + } else { + throw new RestException("A grokStatement must be provided"); + } + } + + private String getTemporaryGrokRootPath() { + String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY); + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + return new Path(grokTempPath, authentication.getName()).toString(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java index c14ec0c..789c421 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java @@ -19,44 +19,73 @@ package org.apache.metron.rest.service.impl; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.HdfsService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static java.nio.charset.StandardCharsets.UTF_8; @Service public class HdfsServiceImpl implements HdfsService { - @Autowired private Configuration configuration; + @Autowired + public HdfsServiceImpl(Configuration configuration) { + this.configuration = configuration; + } + @Override - public byte[] read(Path path) throws IOException { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration); - return byteArrayOutputStream.toByteArray(); + public List<String> list(Path path) throws RestException { + try { + return Arrays.asList(FileSystem.get(configuration).listStatus(path)).stream().map(fileStatus -> fileStatus.getPath().getName()).collect(Collectors.toList()); + } catch (IOException e) { + throw new RestException(e); + } } @Override - public void write(Path path, byte[] contents) throws IOException { - FSDataOutputStream fsDataOutputStream = FileSystem.get(configuration).create(path, true); - fsDataOutputStream.write(contents); - fsDataOutputStream.close(); + public String read(Path path) throws RestException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try { + IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration); + } catch (FileNotFoundException e) { + return null; + } catch (IOException e) { + throw new RestException(e); + } + return new String(byteArrayOutputStream.toByteArray(), UTF_8); } @Override - public FileStatus[] list(Path path) throws IOException { - return FileSystem.get(configuration).listStatus(path); + public void write(Path path, byte[] contents) throws RestException { + FSDataOutputStream fsDataOutputStream; + try { + fsDataOutputStream = FileSystem.get(configuration).create(path, true); + fsDataOutputStream.write(contents); + fsDataOutputStream.close(); + } catch (IOException e) { + throw new RestException(e); + } } @Override - public boolean delete(Path path, boolean recursive) throws IOException { + public boolean delete(Path path, boolean recursive) throws RestException { + try { return FileSystem.get(configuration).delete(path, recursive); + } catch (IOException e) { + throw new RestException(e); + } } } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java index 8b3dbb7..2bfef89 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java @@ -36,12 +36,16 @@ import java.util.Map; @Service public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService { - @Autowired private ObjectMapper objectMapper; - @Autowired private CuratorFramework client; + @Autowired + public SensorEnrichmentConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) { + this.objectMapper = objectMapper; + this.client = client; + } + @Override public SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException { try { http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java index ab46418..9f984e0 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java @@ -38,12 +38,16 @@ import java.util.Map; @Service public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService { - @Autowired private ObjectMapper objectMapper; - @Autowired private CuratorFramework client; + @Autowired + public SensorIndexingConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) { + this.objectMapper = objectMapper; + this.client = client; + } + @Override public Map<String, Object> save(String name, Map<String, Object> sensorIndexingConfig) throws RestException { try { http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java index cb88708..eddfc8d 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java @@ -17,10 +17,8 @@ */ package org.apache.metron.rest.service.impl; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.curator.framework.CuratorFramework; -import org.apache.hadoop.fs.Path; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.SensorParserConfig; @@ -28,20 +26,15 @@ import org.apache.metron.parsers.interfaces.MessageParser; import org.apache.metron.rest.MetronRestConstants; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.ParseMessageRequest; -import org.apache.metron.rest.service.HdfsService; +import org.apache.metron.rest.service.GrokService; import org.apache.metron.rest.service.SensorParserConfigService; import org.apache.zookeeper.KeeperException; import org.json.simple.JSONObject; import org.reflections.Reflections; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; import java.io.File; -import java.io.FileWriter; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -49,72 +42,40 @@ import java.util.Map; import java.util.Set; import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME; -import static org.apache.metron.rest.MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY; -import static org.apache.metron.rest.MetronRestConstants.GROK_PATH_KEY; -import static org.apache.metron.rest.MetronRestConstants.GROK_PATTERN_LABEL_KEY; -import static org.apache.metron.rest.MetronRestConstants.GROK_STATEMENT_KEY; -import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; @Service public class SensorParserConfigServiceImpl implements SensorParserConfigService { - @Autowired - private Environment environment; - - @Autowired private ObjectMapper objectMapper; private CuratorFramework client; + private GrokService grokService; + @Autowired - public void setClient(CuratorFramework client) { + public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) { + this.objectMapper = objectMapper; this.client = client; + this.grokService = grokService; } - @Autowired - private HdfsService hdfsService; - private Map<String, String> availableParsers; @Override public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException { - String serializedConfig; - if (isGrokConfig(sensorParserConfig)) { - addGrokPathToConfig(sensorParserConfig); - sensorParserConfig.getParserConfig().putIfAbsent(MetronRestConstants.GROK_PATTERN_LABEL_KEY, sensorParserConfig.getSensorTopic().toUpperCase()); - String statement = (String) sensorParserConfig.getParserConfig().remove(MetronRestConstants.GROK_STATEMENT_KEY); - serializedConfig = serialize(sensorParserConfig); - sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_STATEMENT_KEY, statement); - saveGrokStatement(sensorParserConfig); - } else { - serializedConfig = serialize(sensorParserConfig); - } try { - ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), serializedConfig.getBytes(), client); + ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client); } catch (Exception e) { throw new RestException(e); } return sensorParserConfig; } - private String serialize(SensorParserConfig sensorParserConfig) throws RestException { - String serializedConfig; - try { - serializedConfig = objectMapper.writeValueAsString(sensorParserConfig); - } catch (JsonProcessingException e) { - throw new RestException("Could not serialize SensorParserConfig", "Could not serialize " + sensorParserConfig.toString(), e.getCause()); - } - return serializedConfig; - } - @Override public SensorParserConfig findOne(String name) throws RestException { SensorParserConfig sensorParserConfig; try { sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(name, client); - if (isGrokConfig(sensorParserConfig)) { - addGrokStatementToConfig(sensorParserConfig); - } } catch (KeeperException.NoNodeException e) { return null; } catch (Exception e) { @@ -145,7 +106,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService return true; } - private List<String> getAllTypes() throws RestException { + @Override + public List<String> getAllTypes() throws RestException { List<String> types; try { types = client.getChildren().forPath(ConfigurationType.PARSER.getZookeeperRoot()); @@ -190,20 +152,21 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService } else if (sensorParserConfig.getParserClassName() == null) { throw new RestException("SensorParserConfig must have a parserClassName"); } else { - MessageParser<JSONObject> parser = null; + MessageParser<JSONObject> parser; try { parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance(); } catch (Exception e) { throw new RestException(e.toString(), e.getCause()); } + File temporaryGrokFile = null; if (isGrokConfig(sensorParserConfig)) { - saveTemporaryGrokStatement(sensorParserConfig); - sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic()).toString()); + temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic()); + sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString()); } parser.configure(sensorParserConfig.getParserConfig()); JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0); - if (isGrokConfig(sensorParserConfig)) { - deleteTemporaryGrokStatement(sensorParserConfig); + if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) { + temporaryGrokFile.delete(); } return results; } @@ -212,78 +175,4 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService private boolean isGrokConfig(SensorParserConfig sensorParserConfig) { return GROK_CLASS_NAME.equals(sensorParserConfig.getParserClassName()); } - - private void addGrokStatementToConfig(SensorParserConfig sensorParserConfig) throws RestException { - String grokStatement = ""; - String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY); - if (grokPath != null) { - String fullGrokStatement = getGrokStatement(grokPath); - String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY); - grokStatement = fullGrokStatement.replaceFirst(patternLabel + " ", ""); - } - sensorParserConfig.getParserConfig().put(GROK_STATEMENT_KEY, grokStatement); - } - - private void addGrokPathToConfig(SensorParserConfig sensorParserConfig) { - if (sensorParserConfig.getParserConfig().get(GROK_PATH_KEY) == null) { - String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY); - if (grokStatement != null) { - sensorParserConfig.getParserConfig().put(GROK_PATH_KEY, - new Path(environment.getProperty(GROK_DEFAULT_PATH_SPRING_PROPERTY), sensorParserConfig.getSensorTopic()).toString()); - } - } - } - - private String getGrokStatement(String path) throws RestException { - try { - return new String(hdfsService.read(new Path(path))); - } catch (IOException e) { - throw new RestException(e); - } - } - - private void saveGrokStatement(SensorParserConfig sensorParserConfig) throws RestException { - saveGrokStatement(sensorParserConfig, false); - } - - private void saveTemporaryGrokStatement(SensorParserConfig sensorParserConfig) throws RestException { - saveGrokStatement(sensorParserConfig, true); - } - - private void saveGrokStatement(SensorParserConfig sensorParserConfig, boolean isTemporary) throws RestException { - String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY); - String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY); - String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY); - if (grokStatement != null) { - String fullGrokStatement = patternLabel + " " + grokStatement; - try { - if (!isTemporary) { - hdfsService.write(new Path(grokPath), fullGrokStatement.getBytes()); - } else { - File grokDirectory = new File(getTemporaryGrokRootPath()); - if (!grokDirectory.exists()) { - grokDirectory.mkdirs(); - } - FileWriter fileWriter = new FileWriter(new File(grokDirectory, sensorParserConfig.getSensorTopic())); - fileWriter.write(fullGrokStatement); - fileWriter.close(); - } - } catch (IOException e) { - throw new RestException(e); - } - } else { - throw new RestException("A grokStatement must be provided"); - } - } - - private void deleteTemporaryGrokStatement(SensorParserConfig sensorParserConfig) { - File file = new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic()); - file.delete(); - } - - private String getTemporaryGrokRootPath() { - String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY); - Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); - return new Path(grokTempPath, authentication.getName()).toString(); - } } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java new file mode 100644 index 0000000..f5392be --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.common.dsl.StellarFunctionInfo; +import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver; +import org.apache.metron.common.field.transformation.FieldTransformations; +import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.rest.model.StellarFunctionDescription; +import org.apache.metron.rest.model.SensorParserContext; +import org.apache.metron.rest.service.StellarService; +import org.json.simple.JSONObject; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +public class StellarServiceImpl implements StellarService { + + @Override + public Map<String, Boolean> validateRules(List<String> rules) { + Map<String, Boolean> results = new HashMap<>(); + StellarProcessor stellarProcessor = new StellarProcessor(); + for(String rule: rules) { + try { + boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT()); + results.put(rule, result); + } catch (ParseException e) { + results.put(rule, false); + } + } + return results; + } + + @Override + public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) { + JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData()); + sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> { + fieldTransformer.transformAndUpdate(sampleJson, sensorParserContext.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT()); + } + ); + return sampleJson; + } + + @Override + public FieldTransformations[] getTransformations() { + return FieldTransformations.values(); + } + + @Override + public List<StellarFunctionDescription> getStellarFunctions() { + List<StellarFunctionDescription> stellarFunctionDescriptions = new ArrayList<>(); + Iterable<StellarFunctionInfo> stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo(); + stellarFunctionsInfo.forEach(stellarFunctionInfo -> { + stellarFunctionDescriptions.add(new StellarFunctionDescription( + stellarFunctionInfo.getName(), + stellarFunctionInfo.getDescription(), + stellarFunctionInfo.getParams(), + stellarFunctionInfo.getReturns())); + }); + return stellarFunctionDescriptions; + } + + @Override + public List<StellarFunctionDescription> getSimpleStellarFunctions() { + List<StellarFunctionDescription> stellarFunctionDescriptions = getStellarFunctions(); + return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription -> + stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList()); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java index cb7c449..9bd368f 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java @@ -33,16 +33,17 @@ public class StormAdminServiceImpl implements StormAdminService { private StormCLIWrapper stormCLIClientWrapper; + private GlobalConfigService globalConfigService; + + private SensorParserConfigService sensorParserConfigService; + @Autowired - public void setStormCLIClientWrapper(StormCLIWrapper stormCLIClientWrapper) { + public StormAdminServiceImpl(StormCLIWrapper stormCLIClientWrapper, GlobalConfigService globalConfigService, SensorParserConfigService sensorParserConfigService) { this.stormCLIClientWrapper = stormCLIClientWrapper; + this.globalConfigService = globalConfigService; + this.sensorParserConfigService = sensorParserConfigService; } - @Autowired - private GlobalConfigService globalConfigService; - - @Autowired - private SensorParserConfigService sensorParserConfigService; @Override public TopologyResponse startParserTopology(String name) throws RestException {
