Repository: incubator-metron Updated Branches: refs/heads/master e416a7d78 -> e66284993
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java new file mode 100644 index 0000000..dd21095 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.mock; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.service.impl.StormCLIWrapper; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class MockStormCLIClientWrapper extends StormCLIWrapper { + + private final Map<String, TopologyStatusCode> parsersStatus = new HashMap<>(); + private TopologyStatusCode enrichmentStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + private TopologyStatusCode indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + + public Set<String> getParserTopologyNames() { + return parsersStatus.keySet(); + } + + public TopologyStatusCode getParserStatus(String name) { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == null) { + return TopologyStatusCode.TOPOLOGY_NOT_FOUND; + } else { + return parserStatus; + } + } + + @Override + public int startParserTopology(String name) throws RestException { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == null || parserStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + parsersStatus.put(name, TopologyStatusCode.ACTIVE); + return 0; + } else { + return 1; + } + } + + @Override + public int stopParserTopology(String name, boolean stopNow) throws RestException { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == TopologyStatusCode.ACTIVE) { + parsersStatus.put(name, TopologyStatusCode.TOPOLOGY_NOT_FOUND); + return 0; + } else { + return 1; + } + } + + public int activateParserTopology(String name) { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == TopologyStatusCode.INACTIVE || parserStatus == TopologyStatusCode.ACTIVE) { + parsersStatus.put(name, TopologyStatusCode.ACTIVE); + return 0; + } else { + return 1; + } + } + + public int deactivateParserTopology(String name) { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == TopologyStatusCode.INACTIVE || parserStatus == TopologyStatusCode.ACTIVE) { + parsersStatus.put(name, TopologyStatusCode.INACTIVE); + return 0; + } else { + return 1; + } + } + + public TopologyStatusCode getEnrichmentStatus() { + return enrichmentStatus; + } + + @Override + public int startEnrichmentTopology() throws RestException { + if (enrichmentStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + enrichmentStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + @Override + public int stopEnrichmentTopology(boolean stopNow) throws RestException { + if (enrichmentStatus == TopologyStatusCode.ACTIVE) { + enrichmentStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + return 0; + } else { + return 1; + } + } + + public int activateEnrichmentTopology() { + if (enrichmentStatus == TopologyStatusCode.INACTIVE || enrichmentStatus == TopologyStatusCode.ACTIVE) { + enrichmentStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + public int deactivateEnrichmentTopology() { + if (enrichmentStatus == TopologyStatusCode.INACTIVE || enrichmentStatus == TopologyStatusCode.ACTIVE) { + enrichmentStatus = TopologyStatusCode.INACTIVE; + return 0; + } else { + return 1; + } + } + + public TopologyStatusCode getIndexingStatus() { + return indexingStatus; + } + + @Override + public int startIndexingTopology() throws RestException { + if (indexingStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + indexingStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + @Override + public int stopIndexingTopology(boolean stopNow) throws RestException { + if (indexingStatus == TopologyStatusCode.ACTIVE) { + indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + return 0; + } else { + return 1; + } + } + + public int activateIndexingTopology() { + if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) { + indexingStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + public int deactivateIndexingTopology() { + if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) { + indexingStatus = TopologyStatusCode.INACTIVE; + return 0; + } else { + return 1; + } + } + + @Override + protected String stormClientVersionInstalled() throws RestException { + return "1.0.1"; + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java new file mode 100644 index 0000000..ca5c1c9 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.mock; + +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.model.TopologyStatus; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.model.TopologySummary; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MockStormRestTemplate extends RestTemplate { + + @Autowired + private Environment environment; + + private MockStormCLIClientWrapper mockStormCLIClientWrapper; + + public void setMockStormCLIClientWrapper(MockStormCLIClientWrapper mockStormCLIClientWrapper) { + this.mockStormCLIClientWrapper = mockStormCLIClientWrapper; + } + + @Override + public Object getForObject(String url, Class responseType, Object... urlVariables) throws RestClientException { + Object response = null; + if (url.equals("http://"; + environment.getProperty(MetronRestConstants.STORM_UI_SPRING_PROPERTY) + MetronRestConstants.TOPOLOGY_SUMMARY_URL)) { + TopologySummary topologySummary = new TopologySummary(); + List<TopologyStatus> topologyStatusList = new ArrayList<>(); + for(String name: mockStormCLIClientWrapper.getParserTopologyNames()) { + topologyStatusList.add(getTopologyStatus(name)); + } + TopologyStatusCode enrichmentStatus = mockStormCLIClientWrapper.getEnrichmentStatus(); + if (enrichmentStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + topologyStatusList.add(getTopologyStatus("enrichment")); + } + TopologyStatusCode indexingStatus = mockStormCLIClientWrapper.getIndexingStatus(); + if (indexingStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + topologyStatusList.add(getTopologyStatus("indexing")); + } + topologySummary.setTopologies(topologyStatusList.toArray(new TopologyStatus[topologyStatusList.size()])); + response = topologySummary; + } else if (url.startsWith("http://"; + environment.getProperty(MetronRestConstants.STORM_UI_SPRING_PROPERTY) + MetronRestConstants.TOPOLOGY_URL + "/")){ + String name = url.substring(url.lastIndexOf('/') + 1, url.length()).replaceFirst("-id", ""); + response = getTopologyStatus(name); + } + return response; + } + + private TopologyStatus getTopologyStatus(String name) { + TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setName(name); + topologyStatus.setId(name + "-id"); + if ("enrichment".equals(name)) { + topologyStatus.setStatus(mockStormCLIClientWrapper.getEnrichmentStatus()); + } else if ("indexing".equals(name)) { + topologyStatus.setStatus(mockStormCLIClientWrapper.getIndexingStatus()); + } else { + topologyStatus.setStatus(mockStormCLIClientWrapper.getParserStatus(name)); + } + return topologyStatus; + } + + @Override + public Object postForObject(String url, Object request, Class responseType, Object... uriVariables) throws RestClientException { + Map<String, String> result = new HashMap<>(); + String[] urlParts = url.split("/"); + String name = urlParts[urlParts.length - 2].replaceFirst("-id", ""); + String action = urlParts[urlParts.length - 1]; + int returnCode = 0; + if (action.equals("activate")) { + if (name.equals("enrichment")) { + returnCode = mockStormCLIClientWrapper.activateEnrichmentTopology(); + } else if (name.equals("indexing")) { + returnCode = mockStormCLIClientWrapper.activateIndexingTopology(); + } else { + returnCode = mockStormCLIClientWrapper.activateParserTopology(name); + } + } else if (action.equals("deactivate")){ + if (name.equals("enrichment")) { + returnCode = mockStormCLIClientWrapper.deactivateEnrichmentTopology(); + } else if (name.equals("indexing")) { + returnCode = mockStormCLIClientWrapper.deactivateIndexingTopology(); + } else { + returnCode = mockStormCLIClientWrapper.deactivateParserTopology(name); + } + } + if (returnCode == 0) { + result.put("status", "success"); + } else { + result.put("status", "error"); + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java new file mode 100644 index 0000000..f7e43ab --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.config.HadoopConfig; +import org.apache.metron.rest.service.impl.HdfsServiceImpl; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.File; +import java.io.IOException; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes={HadoopConfig.class, HdfsServiceTest.HdfsServiceTestContextConfiguration.class}) +@ActiveProfiles(TEST_PROFILE) +public class HdfsServiceTest { + + @Configuration + @Profile("test") + static class HdfsServiceTestContextConfiguration { + + @Bean + public HdfsService hdfsService() { + return new HdfsServiceImpl(); + } + } + + @Autowired + private HdfsService hdfsService; + + @Test + public void test() throws IOException { + String rootDir = "./src/test/tmp"; + File rootFile = new File(rootDir); + Path rootPath = new Path(rootDir); + if (rootFile.exists()) { + FileUtils.cleanDirectory(rootFile); + FileUtils.deleteDirectory(rootFile); + } + assertEquals(true, rootFile.mkdir()); + String fileName1 = "fileName1"; + String fileName2 = "fileName2"; + Path path1 = new Path(rootDir, fileName1); + String value1 = "value1"; + String value2 = "value2"; + Path path2 = new Path(rootDir, fileName2); + String invalidFile = "invalidFile"; + Path pathInvalidFile = new Path(rootDir, invalidFile); + + FileStatus[] fileStatuses = hdfsService.list(new Path(rootDir)); + assertEquals(0, fileStatuses.length); + + + hdfsService.write(path1, value1.getBytes()); + assertEquals(value1, FileUtils.readFileToString(new File(rootDir, fileName1))); + assertEquals(value1, new String(hdfsService.read(path1))); + + fileStatuses = hdfsService.list(rootPath); + assertEquals(1, fileStatuses.length); + assertEquals(fileName1, fileStatuses[0].getPath().getName()); + + hdfsService.write(path2, value2.getBytes()); + assertEquals(value2, FileUtils.readFileToString(new File(rootDir, fileName2))); + assertEquals(value2, new String(hdfsService.read(path2))); + + fileStatuses = hdfsService.list(rootPath); + assertEquals(2, fileStatuses.length); + assertEquals(fileName1, fileStatuses[0].getPath().getName()); + assertEquals(fileName1, fileStatuses[0].getPath().getName()); + + assertEquals(true, hdfsService.delete(path1, false)); + fileStatuses = hdfsService.list(rootPath); + assertEquals(1, fileStatuses.length); + assertEquals(fileName2, fileStatuses[0].getPath().getName()); + assertEquals(true, hdfsService.delete(path2, false)); + fileStatuses = hdfsService.list(rootPath); + assertEquals(0, fileStatuses.length); + + try { + hdfsService.read(pathInvalidFile); + fail("Exception should be thrown when reading invalid file name"); + } catch(IOException e) { + } + assertEquals(false, hdfsService.delete(pathInvalidFile, false)); + + FileUtils.deleteDirectory(new File(rootDir)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java new file mode 100644 index 0000000..561548e --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.GetChildrenBuilder; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.rest.service.impl.SensorParserConfigServiceImpl; +import org.apache.zookeeper.KeeperException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.verifyStatic; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ConfigurationsUtils.class}) +public class SensorParserConfigTest { + + @Mock + private GetChildrenBuilder getChildrenBuilder; + + @Mock + private DeleteBuilder deleteBuilder; + + @Mock + private ObjectMapper objectMapper; + + @Mock + private CuratorFramework client; + + @Mock + private GrokService grokService; + + @InjectMocks + private SensorParserConfigServiceImpl sensorParserConfigService; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + Mockito.when(client.getChildren()).thenReturn(getChildrenBuilder); + Mockito.when(client.delete()).thenReturn(deleteBuilder); + + } + + @Test + public void test() throws Exception { + mockStatic(ConfigurationsUtils.class); + SensorParserConfig broParserConfig = new SensorParserConfig(); + broParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser"); + broParserConfig.setSensorTopic("broTest"); + Mockito.when(objectMapper.writeValueAsString(broParserConfig)).thenReturn(new String(JSONUtils.INSTANCE.toJSON(broParserConfig))); + sensorParserConfigService.save(broParserConfig); + verifyStatic(times(1)); + ConfigurationsUtils.writeSensorParserConfigToZookeeper("broTest", JSONUtils.INSTANCE.toJSON(broParserConfig), client); + + PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("broTest", client)).thenReturn(broParserConfig); + assertEquals(broParserConfig, sensorParserConfigService.findOne("broTest")); + + SensorParserConfig squidParserConfig = new SensorParserConfig(); + squidParserConfig.setParserClassName("org.apache.metron.parsers.GrokParser"); + squidParserConfig.setSensorTopic("squid"); + PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("squidTest", client)).thenReturn(squidParserConfig); + + List<String> allTypes = new ArrayList<String>() {{ + add("broTest"); + add("squidTest"); + }}; + Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenReturn(allTypes); + assertEquals(new ArrayList<SensorParserConfig>() {{ add(broParserConfig); add(squidParserConfig); }}, sensorParserConfigService.getAll()); + + Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(new KeeperException.NoNodeException()); + assertEquals(new ArrayList<>(), sensorParserConfigService.getAll()); + + assertTrue(sensorParserConfigService.delete("broTest")); + verify(deleteBuilder, times(1)).forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest"); + Mockito.when(deleteBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest")).thenThrow(new KeeperException.NoNodeException()); + assertFalse(sensorParserConfigService.delete("broTest")); + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java new file mode 100644 index 0000000..d9522f2 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.utils; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.runtime.RuntimeConstants; +import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader; +import org.reflections.Reflections; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.io.FileWriter; +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; +import java.security.Principal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class ReadMeUtils { + + public static void main(String[] args) throws Exception { + String path; + if (args.length == 0) { + System.out.println("README output path was not set. Defaulting to 'metron-interface/metron-rest/README.md'"); + path = "metron-interface/metron-rest/README.md"; + } else { + path = args[0]; + } + + Reflections reflections = new Reflections("org.apache.metron.rest.controller"); + List<RestControllerInfo> endpoints = new ArrayList<>(); + Set<Class<?>> controllers = reflections.getTypesAnnotatedWith(RestController.class); + for(Class<?> controller: controllers) { + RequestMapping controllerRequestMapping = controller.getAnnotation(RequestMapping.class); + String pathPrefix; + if (controllerRequestMapping == null) { + pathPrefix = ""; + } else { + pathPrefix = controllerRequestMapping.value()[0]; + } + for(Method method: controller.getDeclaredMethods()) { + RestControllerInfo restControllerInfo = new RestControllerInfo(); + RequestMapping requestMapping = method.getAnnotation(RequestMapping.class); + if (requestMapping == null) { + throw new Exception(String.format("@RequestMapping annotation missing from method %s.%s. Every controller method must have a @RequestMapping annotation.", controller.getSimpleName(), method.getName())); + } + ApiOperation apiOperation = method.getAnnotation(ApiOperation.class); + if (apiOperation == null) { + throw new Exception(String.format("@ApiOperation annotation missing from method %s.%s. Every controller method must have an @ApiOperation annotation.", controller.getSimpleName(), method.getName())); + } + ApiResponse[] apiResponseArray; + ApiResponses apiResponses = method.getAnnotation(ApiResponses.class); + if (apiResponses == null) { + ApiResponse apiResponse = method.getAnnotation(ApiResponse.class); + if (apiResponse == null) { + throw new Exception(String.format("@ApiResponses or @ApiResponse annotation missing from method %s.%s. Every controller method must have an @ApiResponses or @ApiResponse annotation.", controller.getSimpleName(), method.getName())); + } else { + apiResponseArray = new ApiResponse[]{ apiResponse }; + } + } else { + apiResponseArray = apiResponses.value(); + } + String[] requestMappingValue = requestMapping.value(); + if (requestMappingValue.length == 0) { + restControllerInfo.setPath(pathPrefix); + } else { + restControllerInfo.setPath(pathPrefix + requestMappingValue[0]); + } + restControllerInfo.setDescription(apiOperation.value()); + RequestMethod requestMethod; + if (requestMapping.method().length == 0) { + requestMethod = RequestMethod.GET; + } else { + requestMethod = requestMapping.method()[0]; + } + restControllerInfo.setMethod(requestMethod); + for (ApiResponse apiResponse: apiResponseArray) { + restControllerInfo.addResponse(apiResponse.message(), apiResponse.code()); + } + for(Parameter parameter: method.getParameters()) { + if (!parameter.getType().equals(Principal.class)) { + ApiParam apiParam = parameter.getAnnotation(ApiParam.class); + if (apiParam == null) { + throw new Exception(String.format("@ApiParam annotation missing from parameter %s.%s.%s. Every controller method parameter must have an @ApiParam annotation.", controller.getSimpleName(), method.getName(), parameter.getName())); + } + restControllerInfo.addParameterDescription(apiParam.name(), apiParam.value()); + } + } + endpoints.add(restControllerInfo); + } + } + Collections.sort(endpoints, (o1, o2) -> o1.getPath().compareTo(o2.getPath())); + + Velocity.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath"); + Velocity.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName()); + Velocity.init(); + + VelocityContext context = new VelocityContext(); + context.put( "endpoints", endpoints ); + + Template template = Velocity.getTemplate("README.vm"); + FileWriter fileWriter = new FileWriter(path); + template.merge( context, fileWriter ); + fileWriter.close(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java new file mode 100644 index 0000000..5b7ac0e --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.utils; + +import org.springframework.web.bind.annotation.RequestMethod; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RestControllerInfo { + + public class Response { + + private String message; + private int code; + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + } + + private String path; + private String description; + private RequestMethod method; + private List<Response> responses = new ArrayList<>(); + private Map<String, String> parameterDescriptions = new HashMap<>(); + + public RestControllerInfo() {} + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public RequestMethod getMethod() { + return method; + } + + public void setMethod(RequestMethod method) { + this.method = method; + } + + public List<Response> getResponses() { + return responses; + } + + public void addResponse(String message, int code) { + Response response = new Response(); + response.setMessage(message); + response.setCode(code); + this.responses.add(response); + } + + public Map<String, String> getParameterDescriptions() { + return parameterDescriptions; + } + + public void addParameterDescription(String name, String description) { + parameterDescriptions.put(name, description); + } +} + + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/resources/README.vm ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/resources/README.vm b/metron-interface/metron-rest/src/test/resources/README.vm new file mode 100644 index 0000000..f259a91 --- /dev/null +++ b/metron-interface/metron-rest/src/test/resources/README.vm @@ -0,0 +1,92 @@ +#[[#]]# Metron REST and Configuration UI + +This UI exposes and aids in sensor configuration. + +#[[##]]# Prerequisites + +* A running Metron cluster +* A running instance of MySQL +* Java 8 installed +* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed + +#[[##]]# Installation +1. Package the Application with Maven: + ``` + mvn clean package + ``` + +1. Untar the archive in the target directory. The directory structure will look like: + ``` + bin + start.sh + lib + metron-rest-version.jar + ``` + +1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/). Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory. + ``` + export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final + ``` + +1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/). Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory. + ``` + export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40 + ``` + +1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html). + +1. Create a Config UI database in MySQL with this command: + ``` + CREATE DATABASE IF NOT EXISTS metronrest + ``` + +1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4. + +1. Start the UI with this command: + ``` + ./bin/start.sh /path/to/application.yml + ``` + +#[[##]]# Usage + +The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. Users can be added with this SQL statement: +``` +use metronrest; +insert into users (username, password, enabled) values ('your_username','your_password',1); +insert into authorities (username, authority) values ('your_username', 'ROLE_USER'); +``` +Users can be added to additional groups with this SQL statement: +``` +use metronrest; +insert into authorities (username, authority) values ('your_username', 'your_group'); +``` + +#[[##]]# API + +Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI. + +| | +| ---------- | +#foreach( $restControllerInfo in $endpoints ) +| [ `$restControllerInfo.getMethod().toString() $restControllerInfo.getPath()`](#$restControllerInfo.getMethod().toString().toLowerCase()-$restControllerInfo.getPath().toLowerCase().replaceAll("/", ""))| +#end + +#foreach( $restControllerInfo in $endpoints ) +#[[###]]# `$restControllerInfo.getMethod().toString() $restControllerInfo.getPath()` + * Description: $restControllerInfo.getDescription() +#if($restControllerInfo.getParameterDescriptions().size() > 0) + * Input: +#end +#foreach( $parameterDescription in $restControllerInfo.getParameterDescriptions().entrySet()) + * $parameterDescription.getKey() - $parameterDescription.getValue() +#end + * Returns: +#foreach( $response in $restControllerInfo.getResponses()) + * $response.getCode() - $response.getMessage() +#end + +#end + +#[[##]]# License + +This project depends on the Java Transaction API. See https://java.net/projects/jta-spec/ for more details. http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/resources/log4j.properties b/metron-interface/metron-rest/src/test/resources/log4j.properties new file mode 100644 index 0000000..492cecf --- /dev/null +++ b/metron-interface/metron-rest/src/test/resources/log4j.properties @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=ERROR, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/pom.xml b/metron-interface/pom.xml new file mode 100644 index 0000000..078da19 --- /dev/null +++ b/metron-interface/pom.xml @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <artifactId>metron-interface</artifactId> + <packaging>pom</packaging> + <name>metron-interface</name> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>Metron</artifactId> + <version>0.3.0</version> + </parent> + <description>Interfaces for Metron</description> + <url>https://metron.incubator.apache.org/</url> + <scm> + <connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-metron.git</connection> + <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-metron.git</developerConnection> + <tag>HEAD</tag> + <url>https://git-wip-us.apache.org/repos/asf/incubator-metron</url> + </scm> + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + </license> + </licenses> + <modules> + <module>metron-rest</module> + <module>metron-rest-client</module> + </modules> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.adrianwalker</groupId> + <artifactId>multiline-string</artifactId> + <version>0.1.2</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + </plugins> + </build> + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-pmd-plugin</artifactId> + <version>3.3</version> + <configuration> + <targetJdk>${global_java_version}</targetJdk> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>emma-maven-plugin</artifactId> + <version>1.0-alpha-3</version> + <inherited>true</inherited> + </plugin> + </plugins> + </reporting> + <repositories> + <repository> + <id>multiline-release-repo</id> + <url>https://raw.github.com/benelog/multiline/master/maven-repository</url> + <snapshots> + <enabled>false</enabled> + </snapshots> + </repository> + </repositories> +</project> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index db28adb..3a8eb9b 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ <module>metron-platform</module> <module>metron-deployment</module> <module>metron-docker</module> + <module>metron-interface</module> </modules> <repositories>
