Repository: metron Updated Branches: refs/heads/master 383d798c7 -> 358e40bd1
METRON-1281 Remove hard-coded indices from the Alerts UI (merrimanr) closes apache/metron#821 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/358e40bd Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/358e40bd Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/358e40bd Branch: refs/heads/master Commit: 358e40bd17d5de2a380a91115eaee9ebd3a453f6 Parents: 383d798 Author: merrimanr <[email protected]> Authored: Thu Nov 2 12:33:48 2017 -0500 Committer: merrimanr <[email protected]> Committed: Thu Nov 2 12:33:48 2017 -0500 ---------------------------------------------------------------------- .../metron-alerts/src/app/utils/constants.ts | 2 +- .../apache/metron/rest/MetronRestConstants.java | 1 + .../rest/service/impl/SearchServiceImpl.java | 19 +++- .../src/main/resources/application.yml | 3 + .../GlobalConfigControllerIntegrationTest.java | 10 +- .../SearchControllerIntegrationTest.java | 49 +++++++++- .../service/impl/SearchServiceImplTest.java | 98 ++++++++++++++++++++ .../org/apache/metron/common/Constants.java | 1 + .../apache/metron/common/error/MetronError.java | 3 +- 9 files changed, 175 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-interface/metron-alerts/src/app/utils/constants.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/utils/constants.ts b/metron-interface/metron-alerts/src/app/utils/constants.ts index 156c65f..b0a5a17 100644 --- a/metron-interface/metron-alerts/src/app/utils/constants.ts +++ b/metron-interface/metron-alerts/src/app/utils/constants.ts @@ -34,4 +34,4 @@ export let CUSTOMM_DATE_RANGE_LABEL = 'Date Range'; export let TREE_SUB_GROUP_SIZE = 5; export let DEFAULT_FACETS = ['source:type', 'ip_src_addr', 'ip_dst_addr', 'host', 'enrichments:geo:ip_dst_addr:country']; export let DEFAULT_GROUPS = ['source:type', 'ip_src_addr', 'ip_dst_addr', 'host', 'enrichments:geo:ip_dst_addr:country']; -export let INDEXES = environment.indices ? environment.indices.split(',') : ['websphere', 'snort', 'asa', 'bro', 'yaf']; +export let INDEXES = environment.indices ? environment.indices.split(',') : []; http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index 0bd0ea3..4e8d7f2 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -59,6 +59,7 @@ public class MetronRestConstants { public static final String SEARCH_MAX_GROUPS = "search.max.groups"; public static final String INDEX_DAO_IMPL = "index.dao.impl"; public static final String INDEX_HBASE_TABLE_PROVIDER_IMPL = "index.hbase.provider"; + public static final String INDEX_WRITER_NAME = "index.writer.name"; public static final String META_DAO_IMPL = "meta.dao.impl"; public static final String META_DAO_SORT = "meta.dao.sort"; http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java index 326ee02..efd80a7 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@ -17,6 +17,11 @@ */ package org.apache.metron.rest.service.impl; +import static org.apache.metron.common.Constants.ERROR_TYPE; +import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; +import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; + +import com.google.common.collect.Lists; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; @@ -27,6 +32,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.SearchService; +import org.apache.metron.rest.service.SensorIndexingConfigService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; @@ -40,16 +46,27 @@ import java.util.List; public class SearchServiceImpl implements SearchService { private IndexDao dao; private Environment environment; + private SensorIndexingConfigService sensorIndexingConfigService; @Autowired - public SearchServiceImpl(IndexDao dao, Environment environment) { + public SearchServiceImpl(IndexDao dao, Environment environment, SensorIndexingConfigService sensorIndexingConfigService) { this.dao = dao; this.environment = environment; + this.sensorIndexingConfigService = sensorIndexingConfigService; } @Override public SearchResponse search(SearchRequest searchRequest) throws RestException { try { + // Pull the indices from the cache by default + if (searchRequest.getIndices() == null || searchRequest.getIndices().isEmpty()) { + List<String> indices = Lists.newArrayList((sensorIndexingConfigService.getAllIndices(environment.getProperty(INDEX_WRITER_NAME)))); + // metaalerts should be included by default + indices.add(METAALERT_TYPE); + // errors should not be included by default + indices.remove(ERROR_TYPE); + searchRequest.setIndices(indices); + } return dao.search(searchRequest); } catch(InvalidSearchException ise) { http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-interface/metron-rest/src/main/resources/application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml index 439a677..452a91f 100644 --- a/metron-interface/metron-rest/src/main/resources/application.yml +++ b/metron-interface/metron-rest/src/main/resources/application.yml @@ -53,6 +53,9 @@ index: dao: # By default, we use the ElasticsearchDao and HBaseDao for backing updates. impl: org.apache.metron.elasticsearch.dao.ElasticsearchDao,org.apache.metron.indexing.dao.HBaseDao + writer: + # Used to retrieve the index names from the sensor index configurations. Should be either 'elasticsearch' or 'solr'. + name: elasticsearch meta: dao: http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java index abb75b1..4351959 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java @@ -91,8 +91,9 @@ public class GlobalConfigControllerIntegrationTest { public void test() throws Exception { this.globalConfigService.delete(); - this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password))) - .andExpect(status().isNotFound()); + assertEventually(() -> this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isNotFound()) + ); this.mockMvc.perform(post(globalConfigUrl).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(globalJson)) .andExpect(status().isCreated()) @@ -109,8 +110,9 @@ public class GlobalConfigControllerIntegrationTest { this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf())) .andExpect(status().isOk()); - this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf())) - .andExpect(status().isNotFound()); + assertEventually(() -> this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()) + ); this.globalConfigService.delete(); } http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index ca7f209..2c15671 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -18,11 +18,11 @@ package org.apache.metron.rest.controller; import com.google.common.collect.ImmutableMap; -import org.apache.metron.hbase.mock.MockHBaseTableProvider; +import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.indexing.dao.InMemoryDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; import org.apache.metron.indexing.dao.search.FieldType; -import org.apache.metron.rest.service.SearchService; +import org.apache.metron.rest.service.SensorIndexingConfigService; import org.json.simple.parser.ParseException; import org.junit.After; import org.junit.Before; @@ -40,6 +40,7 @@ import org.springframework.web.context.WebApplicationContext; import java.util.HashMap; import java.util.Map; +import static org.apache.metron.integration.utils.TestUtils.assertEventually; import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; import static org.hamcrest.Matchers.hasSize; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; @@ -55,10 +56,25 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @ActiveProfiles(TEST_PROFILE) public class SearchControllerIntegrationTest extends DaoControllerTest { - + /** + * { + * "indices": [], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String defaultQuery; @Autowired - private SearchService searchService; + private SensorIndexingConfigService sensorIndexingConfigService; @Autowired private WebApplicationContext wac; @@ -92,6 +108,31 @@ public class SearchControllerIntegrationTest extends DaoControllerTest { } @Test + public void testDefaultQuery() throws Exception { + sensorIndexingConfigService.save("bro", new HashMap<String, Object>() {{ + put("index", "bro"); + }}); + + assertEventually(() -> this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(defaultQuery)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.total").value(5)) + .andExpect(jsonPath("$.results[0].source.source:type").value("bro")) + .andExpect(jsonPath("$.results[0].source.timestamp").value(5)) + .andExpect(jsonPath("$.results[1].source.source:type").value("bro")) + .andExpect(jsonPath("$.results[1].source.timestamp").value(4)) + .andExpect(jsonPath("$.results[2].source.source:type").value("bro")) + .andExpect(jsonPath("$.results[2].source.timestamp").value(3)) + .andExpect(jsonPath("$.results[3].source.source:type").value("bro")) + .andExpect(jsonPath("$.results[3].source.timestamp").value(2)) + .andExpect(jsonPath("$.results[4].source.source:type").value("bro")) + .andExpect(jsonPath("$.results[4].source.timestamp").value(1)) + ); + + sensorIndexingConfigService.delete("bro"); + } + + @Test public void test() throws Exception { this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.allQuery)) http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java new file mode 100644 index 0000000..916422a --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SearchService; +import org.apache.metron.rest.service.SensorIndexingConfigService; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.springframework.core.env.Environment; + +@SuppressWarnings("ALL") +public class SearchServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + IndexDao dao; + Environment environment; + SensorIndexingConfigService sensorIndexingConfigService; + SearchService searchService; + + @Before + public void setUp() throws Exception { + dao = mock(IndexDao.class); + environment = mock(Environment.class); + sensorIndexingConfigService = mock(SensorIndexingConfigService.class); + searchService = new SearchServiceImpl(dao, environment, sensorIndexingConfigService); + } + + + @Test + public void searchShouldProperlySearchDefaultIndices() throws Exception { + when(environment.getProperty(INDEX_WRITER_NAME)).thenReturn("elasticsearch"); + when(sensorIndexingConfigService.getAllIndices("elasticsearch")).thenReturn(Arrays.asList("bro", "snort", "error")); + + SearchRequest searchRequest = new SearchRequest(); + searchService.search(searchRequest); + + SearchRequest expectedSearchRequest = new SearchRequest(); + expectedSearchRequest.setIndices(Arrays.asList("bro", "snort", "metaalert")); + verify(dao).search(eq(expectedSearchRequest)); + + verifyNoMoreInteractions(dao); + } + + @Test + public void searchShouldProperlySearch() throws Exception { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setIndices(Arrays.asList("bro")); + searchService.search(searchRequest); + + SearchRequest expectedSearchRequest = new SearchRequest(); + expectedSearchRequest.setIndices(Arrays.asList("bro")); + verify(dao).search(eq(expectedSearchRequest)); + + verifyNoMoreInteractions(dao); + } + + @Test + public void saveShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + when(dao.search(any(SearchRequest.class))).thenThrow(InvalidSearchException.class); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setIndices(Arrays.asList("bro")); + searchService.search(searchRequest); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java index 8b7e478..b939a92 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java @@ -32,6 +32,7 @@ public class Constants { public static final String ENRICHMENT_TOPIC = "enrichments"; public static final String INDEXING_TOPIC = "indexing"; public static final String ERROR_STREAM = "error"; + public static final String ERROR_TYPE = "error"; public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment"; public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel"; public static final String GUID = "guid"; http://git-wip-us.apache.org/repos/asf/metron/blob/358e40bd/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java index 2837d34..9a553ca 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java @@ -32,13 +32,14 @@ import java.util.Optional; import java.util.Set; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.metron.common.Constants.ERROR_TYPE; import static org.apache.metron.common.Constants.ErrorFields; public class MetronError { private String message; private Throwable throwable; - private String sensorType = "error"; + private String sensorType = ERROR_TYPE; private ErrorType errorType = ErrorType.DEFAULT_ERROR; private Set<String> errorFields; private List<Object> rawMessages;
