Repository: metron Updated Branches: refs/heads/master a25de0fc1 -> 1c12e22df
METRON-1243: Add a REST endpoint which allows us to get a list of all indice closes apache/incubator-metron#797 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c12e22d Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c12e22d Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c12e22d Branch: refs/heads/master Commit: 1c12e22df03b53981ca8cef426ab26f263f289c0 Parents: a25de0f Author: cstella <ceste...@gmail.com> Authored: Wed Oct 25 14:48:17 2017 -0400 Committer: cstella <ceste...@gmail.com> Committed: Wed Oct 25 14:48:17 2017 -0400 ---------------------------------------------------------------------- .../SensorIndexingConfigController.java | 8 ++ .../service/SensorIndexingConfigService.java | 2 + .../impl/SensorIndexingConfigServiceImpl.java | 34 +++++++- ...IndexingConfigControllerIntegrationTest.java | 15 ++++ .../SensorIndexingConfigServiceImplTest.java | 83 +++++++++++++++++++- 5 files changed, 140 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java index ffd80fb..ff1d6bd 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java @@ -32,6 +32,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.util.List; import java.util.Map; @RestController @@ -74,6 +75,13 @@ public class SensorIndexingConfigController { return new ResponseEntity<>(sensorIndexingConfigService.getAll(), HttpStatus.OK); } + @ApiOperation(value = "Retrieves all indices") + @ApiResponse(message = "Returns all the indices in use", code = 200) + @RequestMapping(value = "/list/indices/{writerName}", method = RequestMethod.GET) + ResponseEntity<Iterable<String>> getAllIndices(@ApiParam(name="writerName", value="Writer name. One of solr, elasticsearch or hdfs", required=true)@PathVariable String writerName) throws Exception { + return new ResponseEntity<>(sensorIndexingConfigService.getAllIndices(writerName), HttpStatus.OK); + } + @ApiOperation(value = "Deletes a SensorIndexingConfig from Zookeeper") @ApiResponses(value = { @ApiResponse(message = "SensorIndexingConfig was deleted", code = 200), @ApiResponse(message = "SensorIndexingConfig is missing", code = 404) }) http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java index 0699431..0f32e53 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java @@ -32,6 +32,8 @@ public interface SensorIndexingConfigService { List<String> getAllTypes() throws RestException; + Iterable<String> getAllIndices(String writerName) throws RestException; + boolean delete(String name) throws RestException; } http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java index 5c73b26..a90d123 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java @@ -18,14 +18,16 @@ package org.apache.metron.rest.service.impl; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterables; +import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.IndexingConfigurations; +import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.zookeeper.ConfigurationsCache; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.SensorIndexingConfigService; -import org.apache.metron.common.zookeeper.ZKConfigurationsCache; import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -33,6 +35,9 @@ import org.springframework.stereotype.Service; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Collections; +import java.util.Set; +import java.util.HashSet; @Service public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService { @@ -85,6 +90,33 @@ public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigServ return configs.getTypes(); } + /** + * Get a list of index names for a given writer (e.g. elasticsearch, solr, hdfs). + * This functions in the following way: + * * If an index config exists, then the index name will be returned. If unspecified, then the sensor name is used + * * If a parser exists and an index does NOT exist, then it will be included. + * * If the writer is disabled in the index config, then it will NOT be included. + * @param writerName The writer name to use + * @return An iterable of index names + * @throws RestException + */ + @Override + public Iterable<String> getAllIndices(String writerName) throws RestException { + if(StringUtils.isEmpty(writerName)) { + return Collections.emptyList(); + } + IndexingConfigurations indexingConfigs = cache.get( IndexingConfigurations.class); + ParserConfigurations parserConfigs = cache.get( ParserConfigurations.class); + Set<String> ret = new HashSet<>(); + for(String sensorName : Iterables.concat(parserConfigs.getTypes(), indexingConfigs.getTypes())) { + if(indexingConfigs.isEnabled(sensorName, writerName)) { + String indexName = indexingConfigs.getIndex(sensorName, writerName); + ret.add(indexName == null ? sensorName : indexName); + } + } + return ret; + } + @Override public boolean delete(String name) throws RestException { try { http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java index 674c55a..28977fd 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java @@ -98,12 +98,27 @@ public class SensorIndexingConfigControllerIntegrationTest { .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(content().bytes("{}".getBytes())); + this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(content().bytes("[]".getBytes())); + + this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/blah").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(content().bytes("[]".getBytes())); + this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.index").value("broTest")) .andExpect(jsonPath("$.batchSize").value(1)); + this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(content().bytes("[\"broTest\"]".getBytes())); + assertEventually(() -> this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java index 9641a52..1d8e3f5 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java @@ -17,10 +17,10 @@ */ package org.apache.metron.rest.service.impl; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import org.adrianwalker.multilinestring.Multiline; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.DeleteBuilder; @@ -29,12 +29,14 @@ import org.apache.curator.framework.api.GetDataBuilder; import org.apache.curator.framework.api.SetDataBuilder; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.IndexingConfigurations; +import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.common.zookeeper.ConfigurationsCache; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.SensorIndexingConfigService; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -44,6 +46,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Collections; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -139,7 +142,85 @@ public class SensorIndexingConfigServiceImplTest { } + @Test + public void getAllIndicesWithOnlyParsers() throws RestException { + ParserConfigurations parserConfiguration = mock(ParserConfigurations.class); + when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort")); + IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class); + when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList()); + when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn(null); + when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null); + when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true); + when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true); + + when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration); + when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration); + List<String> indices = new ArrayList<String>(); + Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch")); + Assert.assertEquals(2, indices.size()); + Assert.assertTrue(indices.contains("bro")); + Assert.assertTrue(indices.contains("snort")); + } + + @Test + public void getAllIndicesWithOnlyIndexing() throws RestException { + ParserConfigurations parserConfiguration = mock(ParserConfigurations.class); + when(parserConfiguration.getTypes()).thenReturn(Collections.emptyList()); + IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class); + // rename bro, include snort by default configs, and disable yaf + when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "yaf")); + when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro"); + when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null); + when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true); + when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true); + when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(false); + + when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration); + when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration); + List<String> indices = new ArrayList<String>(); + Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch")); + Assert.assertEquals(2, indices.size()); + Assert.assertTrue(indices.contains("renamed_bro")); + Assert.assertTrue(indices.contains("snort")); + } + @Test + public void getAllIndicesWithParsersAndIndexConfigs() throws RestException { + + ParserConfigurations parserConfiguration = mock(ParserConfigurations.class); + when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "yaf")); + IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class); + when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "squid")); + when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro"); + when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn("snort"); + when(indexingConfiguration.getIndex(eq("yaf"), eq("elasticsearch"))).thenReturn(null); + when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true); + when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true); + when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(true); + when(indexingConfiguration.isEnabled(eq("squid"), eq("elasticsearch"))).thenReturn(false); + when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration); + when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration); + List<String> indices = new ArrayList<String>(); + Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch")); + Assert.assertEquals(3, indices.size()); + Assert.assertTrue(indices.contains("renamed_bro")); + Assert.assertTrue(indices.contains("snort")); + Assert.assertTrue(indices.contains("yaf")); + } + + @Test + public void getAllIndicesWithNoConfigs() throws RestException { + ParserConfigurations parserConfiguration = mock(ParserConfigurations.class); + when(parserConfiguration.getTypes()).thenReturn(Collections.emptyList()); + IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class); + when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList()); + + when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration); + when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration); + List<String> indices = new ArrayList<String>(); + Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch")); + Assert.assertEquals(0, indices.size()); + } @Test public void getAllTypesShouldProperlyReturnTypes() throws Exception {