METRON-1150 REST parseMessage endpoint fails with Unable to load <grok file> from either classpath or HDFS (merrimanr) closes apache/metron#729
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/224d3d5e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/224d3d5e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/224d3d5e Branch: refs/heads/Metron_0.4.1 Commit: 224d3d5e2b8c17ca4414ad7d7846dafee0b06d3b Parents: f79214b Author: merrimanr <[email protected]> Authored: Fri Sep 8 09:43:29 2017 -0500 Committer: merrimanr <[email protected]> Committed: Fri Sep 8 09:43:29 2017 -0500 ---------------------------------------------------------------------- .../CURRENT/configuration/metron-rest-env.xml | 10 +--- .../package/scripts/params/params_linux.py | 6 ++- .../METRON/CURRENT/package/templates/metron.j2 | 1 - .../src/main/config/rest_application.yml | 1 - .../apache/metron/rest/MetronRestConstants.java | 1 - .../apache/metron/rest/service/GrokService.java | 6 ++- .../apache/metron/rest/service/HdfsService.java | 2 + .../rest/service/impl/GrokServiceImpl.java | 37 ++++++------- .../rest/service/impl/HdfsServiceImpl.java | 9 ++++ .../impl/SensorParserConfigServiceImpl.java | 41 +++++++------- .../src/main/resources/application-docker.yml | 1 - .../src/main/resources/application-test.yml | 1 - .../src/main/resources/application-vagrant.yml | 3 +- .../src/main/resources/application.yml | 3 +- ...orParserConfigControllerIntegrationTest.java | 6 --- .../rest/service/impl/GrokServiceImplTest.java | 56 +++++++------------- .../impl/SensorParserConfigServiceImplTest.java | 20 +++++-- 17 files changed, 97 insertions(+), 107 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 9c11123..0549510 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -90,17 +90,11 @@ </property> <property> <name>metron_temp_grok_path</name> - <description>Temporary local file path where grok patterns are written during testing</description> - <value>./patterns/temp</value> + <description>Temporary HDFS file path where grok patterns are written during testing</description> + <value>{{metron_apps_hdfs_dir}}/patterns/tmp</value> <display-name>Metron temp grok path</display-name> </property> <property> - <name>metron_default_grok_path</name> - <description>Default HDFS directory path used when storing Grok patterns</description> - <value>/apps/metron/patterns</value> - <display-name>Metron default grok path</display-name> - </property> - <property> <name>metron_spring_options</name> <description>Additional Spring options not included in the rest_application.yml file</description> <display-name>Metron Spring options</display-name> http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 78d253c..a9d00dd 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -55,8 +55,6 @@ metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_ metron_jdbc_password = config['configurations']['metron-rest-env']['metron_jdbc_password'] metron_jdbc_platform = config['configurations']['metron-rest-env']['metron_jdbc_platform'] metron_jdbc_client_path = config['configurations']['metron-rest-env']['metron_jdbc_client_path'] -metron_temp_grok_path = config['configurations']['metron-rest-env']['metron_temp_grok_path'] -metron_default_grok_path = config['configurations']['metron-rest-env']['metron_default_grok_path'] metron_spring_options = config['configurations']['metron-rest-env']['metron_spring_options'] metron_escalation_topic = config['configurations']['metron-rest-env']['metron_escalation_topic'] metron_config_path = metron_home + '/config' @@ -132,6 +130,10 @@ if has_kafka_host: metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir'] +# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works +# well enough until we find a better way via Ambari +metron_temp_grok_path = format(format(config['configurations']['metron-rest-env']['metron_temp_grok_path'])) + metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention'] local_grok_patterns_dir = format("{metron_home}/patterns") http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index dd37946..049b6ee 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -27,7 +27,6 @@ METRON_JDBC_USERNAME="{{metron_jdbc_username}}" METRON_JDBC_PLATFORM="{{metron_jdbc_platform}}" METRON_JDBC_CLIENT_PATH="{{metron_jdbc_client_path}}" METRON_TEMP_GROK_PATH="{{metron_temp_grok_path}}" -METRON_DEFAULT_GROK_PATH="{{metron_default_grok_path}}" METRON_SPRING_OPTIONS="{{metron_spring_options}}" ZOOKEEPER="{{zookeeper_quorum}}" BROKERLIST="{{kafka_brokers}}" http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/config/rest_application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml index 0c17580..6e4fb66 100644 --- a/metron-interface/metron-rest/src/main/config/rest_application.yml +++ b/metron-interface/metron-rest/src/main/config/rest_application.yml @@ -36,7 +36,6 @@ kafka: grok: path: temp: ${METRON_TEMP_GROK_PATH} - default: ${METRON_DEFAULT_GROK_PATH} storm: ui: http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index 7c9cdac..7f8110c 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -26,7 +26,6 @@ public class MetronRestConstants { public static final String DOCKER_PROFILE = "docker"; public static final String CSRF_ENABLE_PROFILE = "csrf-enable"; - public static final String GROK_DEFAULT_PATH_SPRING_PROPERTY = "grok.path.default"; public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp"; public static final String GROK_CLASS_NAME = GrokParser.class.getName(); public static final String GROK_PATH_KEY = "grokPath"; http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java index adeb1ed..807a330 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java @@ -17,10 +17,10 @@ */ package org.apache.metron.rest.service; +import org.apache.hadoop.fs.Path; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.GrokValidation; -import java.io.File; import java.util.Map; public interface GrokService { @@ -29,7 +29,9 @@ public interface GrokService { GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException; - File saveTemporary(String statement, String name) throws RestException; + Path saveTemporary(String statement, String name) throws RestException; + + void deleteTemporary() throws RestException; String getStatementFromClasspath(String path) throws RestException; http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java index d5932c7..58dbf9b 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java @@ -31,4 +31,6 @@ public interface HdfsService { List<String> list(Path path) throws RestException; boolean delete(Path path, boolean recursive) throws RestException; + + boolean mkdirs(Path path) throws RestException; } http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java index edae13b..e185b53 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java @@ -17,6 +17,7 @@ */ package org.apache.metron.rest.service.impl; +import java.nio.charset.Charset; import oi.thekraken.grok.api.Grok; import oi.thekraken.grok.api.Match; import org.apache.commons.io.IOUtils; @@ -25,15 +26,13 @@ import org.apache.hadoop.fs.Path; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.GrokValidation; import org.apache.metron.rest.service.GrokService; +import org.apache.metron.rest.service.HdfsService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; import java.io.InputStreamReader; import java.io.StringReader; import java.util.Map; @@ -47,10 +46,13 @@ public class GrokServiceImpl implements GrokService { private Grok commonGrok; + private HdfsService hdfsService; + @Autowired - public GrokServiceImpl(Environment environment, Grok commonGrok) { + public GrokServiceImpl(Environment environment, Grok commonGrok, HdfsService hdfsService) { this.environment = environment; this.commonGrok = commonGrok; + this.hdfsService = hdfsService; } @Override @@ -85,30 +87,25 @@ public class GrokServiceImpl implements GrokService { } @Override - public File saveTemporary(String statement, String name) throws RestException { + public Path saveTemporary(String statement, String name) throws RestException { if (statement != null) { - try { - File grokDirectory = new File(getTemporaryGrokRootPath()); - if (!grokDirectory.exists()) { - grokDirectory.mkdirs(); - } - File path = new File(grokDirectory, name); - FileWriter fileWriter = new FileWriter(new File(grokDirectory, name)); - fileWriter.write(statement); - fileWriter.close(); - return path; - } catch (IOException e) { - throw new RestException(e); - } + Path path = getTemporaryGrokRootPath(); + hdfsService.mkdirs(path); + hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8"))); + return path; } else { throw new RestException("A grokStatement must be provided"); } } - private String getTemporaryGrokRootPath() { + public void deleteTemporary() throws RestException { + hdfsService.delete(getTemporaryGrokRootPath(), true); + } + + private Path getTemporaryGrokRootPath() { String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY); Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); - return new Path(grokTempPath, authentication.getName()).toString(); + return new Path(grokTempPath, authentication.getName()); } public String getStatementFromClasspath(String path) throws RestException { http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java index 789c421..a9ae8eb 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java @@ -88,4 +88,13 @@ public class HdfsServiceImpl implements HdfsService { throw new RestException(e); } } + + @Override + public boolean mkdirs(Path path) throws RestException { + try { + return FileSystem.get(configuration).mkdirs(path); + } catch (IOException e) { + throw new RestException(e); + } + } } http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java index 37d59d0..f99b41c 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java @@ -17,8 +17,16 @@ */ package org.apache.metron.rest.service.impl; +import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.curator.framework.CuratorFramework; +import org.apache.hadoop.fs.Path; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.SensorParserConfig; @@ -34,15 +42,6 @@ import org.reflections.Reflections; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME; - @Service public class SensorParserConfigServiceImpl implements SensorParserConfigService { @@ -53,7 +52,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService private GrokService grokService; @Autowired - public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) { + public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, + GrokService grokService) { this.objectMapper = objectMapper; this.client = client; this.grokService = grokService; @@ -64,7 +64,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService @Override public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException { try { - ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client); + ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), + objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client); } catch (Exception e) { throw new RestException(e); } @@ -126,7 +127,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService Set<Class<? extends MessageParser>> parserClasses = getParserClasses(); parserClasses.forEach(parserClass -> { if (!"BasicParser".equals(parserClass.getSimpleName())) { - availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""), parserClass.getName()); + availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""), + parserClass.getName()); } }); } @@ -154,20 +156,23 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService } else { MessageParser<JSONObject> parser; try { - parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance(); + parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()) + .newInstance(); } catch (Exception e) { throw new RestException(e.toString(), e.getCause()); } - File temporaryGrokFile = null; + Path temporaryGrokPath = null; if (isGrokConfig(sensorParserConfig)) { - temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic()); - sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString()); + String name = parseMessageRequest.getSensorParserConfig().getSensorTopic(); + temporaryGrokPath = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), name); + sensorParserConfig.getParserConfig() + .put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString()); } parser.configure(sensorParserConfig.getParserConfig()); parser.init(); JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0); - if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) { - temporaryGrokFile.delete(); + if (isGrokConfig(sensorParserConfig) && temporaryGrokPath != null) { + grokService.deleteTemporary(); } return results; } http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-docker.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml index 15fa293..426a0da 100644 --- a/metron-interface/metron-rest/src/main/resources/application-docker.yml +++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml @@ -46,7 +46,6 @@ hdfs: grok: path: temp: target/patterns/temp - default: target/patterns storm: ui: http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-test.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml index 9793840..b5e65a7 100644 --- a/metron-interface/metron-rest/src/main/resources/application-test.yml +++ b/metron-interface/metron-rest/src/main/resources/application-test.yml @@ -29,7 +29,6 @@ spring: grok: path: temp: target/patterns/temp - default: target/patterns storm: ui: http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-vagrant.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml index 31b5784..cf2c170 100644 --- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml +++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml @@ -39,8 +39,7 @@ hdfs: grok: path: - temp: ./patterns/temp - default: /apps/metron/patterns + temp: /apps/metron/patterns/tmp storm: ui: http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml index cf9218b..d96439c 100644 --- a/metron-interface/metron-rest/src/main/resources/application.yml +++ b/metron-interface/metron-rest/src/main/resources/application.yml @@ -26,8 +26,7 @@ spring: grok: path: - temp: ./ - default: /apps/metron/patterns + temp: /apps/metron/patterns/tmp zookeeper: client: http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java index 66771eb..6e2d788 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java @@ -21,7 +21,6 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.io.FileUtils; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.rest.MetronRestConstants; -import org.apache.metron.rest.service.GrokService; import org.apache.metron.rest.service.SensorParserConfigService; import org.junit.Before; import org.junit.Test; @@ -365,11 +364,6 @@ public class SensorParserConfigControllerIntegrationTest { FileUtils.cleanDirectory(grokTempPath); FileUtils.deleteDirectory(grokTempPath); } - File grokPath = new File(environment.getProperty(MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY)); - if (grokPath.exists()) { - FileUtils.cleanDirectory(grokPath); - FileUtils.deleteDirectory(grokPath); - } } } http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java index 1935269..8f7f40c 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java @@ -17,51 +17,46 @@ */ package org.apache.metron.rest.service.impl; +import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Map; import oi.thekraken.grok.api.Grok; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.GrokValidation; import org.apache.metron.rest.service.GrokService; +import org.apache.metron.rest.service.HdfsService; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import org.springframework.core.env.Environment; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.powermock.api.mockito.PowerMockito.when; -import static org.powermock.api.mockito.PowerMockito.whenNew; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({GrokServiceImpl.class, FileWriter.class}) public class GrokServiceImplTest { @Rule public final ExpectedException exception = ExpectedException.none(); private Environment environment; private Grok grok; + private HdfsService hdfsService; private GrokService grokService; @Before public void setUp() throws Exception { environment = mock(Environment.class); grok = mock(Grok.class); - grokService = new GrokServiceImpl(environment, grok); + hdfsService = new HdfsServiceImpl(new Configuration()); + grokService = new GrokServiceImpl(environment, grok, hdfsService); } @Test @@ -205,24 +200,9 @@ public class GrokServiceImplTest { grokService.saveTemporary(statement, "squid"); - File testFile = new File("./target/user1/squid"); - assertEquals(statement, FileUtils.readFileToString(testFile)); - testFile.delete(); - } - - @Test - public void saveTemporaryShouldWrapExceptionInRestException() throws Exception { - exception.expect(RestException.class); - - String statement = "grok statement"; - - Authentication authentication = mock(Authentication.class); - when(authentication.getName()).thenReturn("user1"); - SecurityContextHolder.getContext().setAuthentication(authentication); - when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target"); - whenNew(FileWriter.class).withParameterTypes(File.class).withArguments(any()).thenThrow(new IOException()); - - grokService.saveTemporary(statement, "squid"); + File testRoot = new File("./target/user1"); + assertEquals(statement, FileUtils.readFileToString(new File(testRoot, "squid"), Charset.forName("utf-8"))); + testRoot.delete(); } @Test http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java index d35a48c..c96a796 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java @@ -18,12 +18,14 @@ package org.apache.metron.rest.service.impl; import com.fasterxml.jackson.databind.ObjectMapper; +import oi.thekraken.grok.api.Grok; import org.adrianwalker.multilinestring.Multiline; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.DeleteBuilder; import org.apache.curator.framework.api.GetChildrenBuilder; import org.apache.curator.framework.api.GetDataBuilder; import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.hadoop.conf.Configuration; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.rest.RestException; @@ -43,7 +45,10 @@ import java.io.FileWriter; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -88,11 +93,18 @@ public class SensorParserConfigServiceImplTest { @Multiline public static String broJson; + private String user = "user1"; + @Before public void setUp() throws Exception { objectMapper = mock(ObjectMapper.class); curatorFramework = mock(CuratorFramework.class); - grokService = mock(GrokService.class); + Environment environment = mock(Environment.class); + Authentication authentication = mock(Authentication.class); + when(authentication.getName()).thenReturn(user); + SecurityContextHolder.getContext().setAuthentication(authentication); + when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target"); + grokService = new GrokServiceImpl(environment, mock(Grok.class), new HdfsServiceImpl(new Configuration())); sensorParserConfigService = new SensorParserConfigServiceImpl(objectMapper, curatorFramework, grokService); } @@ -269,13 +281,13 @@ public class SensorParserConfigServiceImplTest { parseMessageRequest.setGrokStatement(grokStatement); parseMessageRequest.setSampleData(sampleData); - File patternFile = new File("./target/squidTest"); + File grokRoot = new File("./target", user); + grokRoot.mkdir(); + File patternFile = new File(grokRoot, "squid"); FileWriter writer = new FileWriter(patternFile); writer.write(grokStatement); writer.close(); - when(grokService.saveTemporary(grokStatement, "squid")).thenReturn(patternFile); - assertEquals(new HashMap() {{ put("elapsed", 161); put("code", 200);
