Repository: metron Updated Branches: refs/heads/master 0d1923f83 -> cf7043c59
http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java new file mode 100644 index 0000000..ae4f9bd --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao.search; + +import java.util.Map; + +public class SearchResult { + + private String id; + private Map<String, Object> source; + private float score; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Map<String, Object> getSource() { + return source; + } + + public void setSource(Map<String, Object> source) { + this.source = source; + } + + public float getScore() { + return score; + } + + public void setScore(float score) { + this.score = score; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java new file mode 100644 index 0000000..a3473fc --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao.search; + +public class SortField { + private String field; + private SortOrder sortOrder; + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + public SortOrder getSortOrder() { + return sortOrder; + } + + public void setSortOrder(String sortOrder) { + this.sortOrder = SortOrder.fromString(sortOrder); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java new file mode 100644 index 0000000..fde3279 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao.search; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum SortOrder { + @JsonProperty("desc") + DESC("desc"), + @JsonProperty("asc") + ASC("asc"); + + private String sortOrder; + + SortOrder(String sortOrder) { + this.sortOrder = sortOrder; + } + + public String getSortOrder() { + return sortOrder; + } + + public static SortOrder fromString(String order) { + return SortOrder.valueOf(order.toUpperCase()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java new file mode 100644 index 0000000..444a9da --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Splitter; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Iterables; +import org.apache.metron.common.Constants; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.search.*; + +import java.io.IOException; +import java.util.*; + +public class InMemoryDao implements IndexDao { + public static Map<String, List<String>> BACKING_STORE = new HashMap<>(); + private AccessConfig config; + + @Override + public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { + if(config.getMaxSearchResults() != null && searchRequest.getSize() > config.getMaxSearchResults()) { + throw new InvalidSearchException("Search result size must be less than " + config.getMaxSearchResults()); + } + List<SearchResult> response = new ArrayList<>(); + for(String index : searchRequest.getIndices()) { + String i = null; + for(String storedIdx : BACKING_STORE.keySet()) { + if(storedIdx.equals(index) || storedIdx.startsWith(index + "_")) { + i = storedIdx; + } + } + if(i == null) { + continue; + } + for (String doc : BACKING_STORE.get(i)) { + Map<String, Object> docParsed = parse(doc); + if (isMatch(searchRequest.getQuery(), docParsed)) { + SearchResult result = new SearchResult(); + result.setSource(docParsed); + result.setScore((float) Math.random()); + result.setId(docParsed.getOrDefault(Constants.GUID, UUID.randomUUID()).toString()); + response.add(result); + } + } + } + + if(searchRequest.getSort().size() != 0) { + Collections.sort(response, sorted(searchRequest.getSort())); + } + SearchResponse ret = new SearchResponse(); + List<SearchResult> finalResp = new ArrayList<>(); + int maxSize = config.getMaxSearchResults() == null?searchRequest.getSize():config.getMaxSearchResults(); + for(int i = searchRequest.getFrom();i < response.size()&& finalResp.size() <= maxSize;++i) { + finalResp.add(response.get(i)); + } + ret.setTotal(response.size()); + ret.setResults(finalResp); + return ret; + } + + + private static class ComparableComparator implements Comparator<Comparable> { + SortOrder order = null; + public ComparableComparator(SortOrder order) { + this.order = order; + } + @Override + public int compare(Comparable o1, Comparable o2) { + int result = ComparisonChain.start().compare(o1, o2).result(); + return order == SortOrder.ASC?result:-1*result; + } + } + + private static Comparator<SearchResult> sorted(final List<SortField> fields) { + return (o1, o2) -> { + ComparisonChain chain = ComparisonChain.start(); + for(SortField field : fields) { + Comparable f1 = (Comparable) o1.getSource().get(field.getField()); + Comparable f2 = (Comparable) o2.getSource().get(field.getField()); + chain = chain.compare(f1, f2, new ComparableComparator(field.getSortOrder())); + } + return chain.result(); + }; + } + + private static boolean isMatch(String query, Map<String, Object> doc) { + if(query.equals("*")) { + return true; + } + if(query.contains(":")) { + Iterable<String> splits = Splitter.on(":").split(query.trim()); + String field = Iterables.getFirst(splits, ""); + String val = Iterables.getLast(splits, ""); + Object o = doc.get(field); + if(o == null) { + return false; + } + else { + return o.equals(val); + } + } + return false; + } + + private static Map<String, Object> parse(String doc) { + try { + return JSONUtils.INSTANCE.load(doc, new TypeReference<Map<String, Object>>() {}); + } catch (IOException e) { + throw new IllegalStateException(e.getMessage(), e); + } + + } + + @Override + public void init(Map<String, Object> globalConfig, AccessConfig config) { + this.config = config; + } + + public static void load(Map<String, List<String>> backingStore) { + BACKING_STORE = backingStore; + } + + public static void clear() { + BACKING_STORE.clear(); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java new file mode 100644 index 0000000..8b5baef --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.integration.InMemoryComponent; +import org.json.simple.parser.ParseException; +import org.junit.*; + +import java.util.List; + +public abstract class IndexingDaoIntegrationTest { + /** + * [ + * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "timestamp":1, "rejected":true}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "timestamp":2, "rejected":false}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "timestamp":3, "rejected":true}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "timestamp":4, "rejected":false}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "timestamp":5, "rejected":true} + * ] + */ + @Multiline + public static String broData; + + /** + * [ + * {"source:type": "snort" "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "timestamp":6, "is_alert":false}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "timestamp":7, "is_alert":true}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "timestamp":8, "is_alert":false}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "timestamp":9, "is_alert":true}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "timestamp":10, "is_alert":false} + * ] + */ + @Multiline + public static String snortData; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String allQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "ip_src_addr:192.168.1.1", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String filterQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "ip_src_port", + * "sortOrder": "asc" + * } + * ] + * } + */ + @Multiline + public static String sortQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 4, + * "size": 3, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String paginationQuery; + + /** + * { + * "indices": ["bro"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String indexQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 101, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String exceededMaxResultsQuery; + + protected IndexDao dao; + protected InMemoryComponent indexComponent; + + @Before + public void setup() throws Exception { + indexComponent = startIndex(); + loadTestData(); + dao = createDao(); + } + + @Test + public void test() throws Exception { + //All Query Testcase + { + SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + for(int i = 0;i < 5;++i) { + Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); + } + for(int i = 5;i < 10;++i) { + Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); + } + } + //Filter test case + { + SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(3, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); + Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); + } + //Sort test case + { + SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + for(int i = 8001;i < 8011;++i) { + Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port")); + } + } + //pagination test case + { + SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals(3, results.size()); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); + Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); + } + //Index query + { + SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(5, response.getTotal()); + List<SearchResult> results = response.getResults(); + for(int i = 5,j=0;i > 0;i--,j++) { + Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); + Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); + } + } + //Exceeded maximum results query + { + SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); + try { + dao.search(request); + Assert.fail("Exception expected, but did not come."); + } + catch(InvalidSearchException ise) { + Assert.assertEquals("Search result size must be less than 100", ise.getMessage()); + } + } + } + + @After + public void stop() throws Exception { + indexComponent.stop(); + } + + protected abstract IndexDao createDao() throws Exception; + protected abstract InMemoryComponent startIndex() throws Exception; + protected abstract void loadTestData() throws Exception; +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 5079655..5d64019 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,7 @@ <global_maven_version>[3.3.1,)</global_maven_version> <global_kryo_version>3.0.3</global_kryo_version> <global_kryo_serializers_version>0.38</global_kryo_serializers_version> + <global_reflections_version>0.9.10</global_reflections_version> <argLine></argLine> </properties>
