Repository: metron Updated Branches: refs/heads/master 7af11b626 -> 28f4b5704
METRON-1660 On Solr, sorting by threat score fails (justinleet) closes apache/metron#1102 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/28f4b570 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/28f4b570 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/28f4b570 Branch: refs/heads/master Commit: 28f4b570493eda0a23317f520b89cb370e606ca0 Parents: 7af11b6 Author: justinleet <[email protected]> Authored: Wed Jul 11 15:48:08 2018 -0400 Committer: leet <[email protected]> Committed: Wed Jul 11 15:48:08 2018 -0400 ---------------------------------------------------------------------- .../dao/metaalert/MetaAlertIntegrationTest.java | 56 ++++++++++++++++++++ .../src/main/config/schema/metaalert/schema.xml | 6 ++- 2 files changed, 61 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/28f4b570/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java index 6f96fb5..f754b81 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java @@ -51,6 +51,7 @@ import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.search.SortField; +import org.apache.metron.indexing.dao.search.SortOrder; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; @@ -194,6 +195,60 @@ public abstract class MetaAlertIntegrationTest { } @Test + public void shouldSortByThreatTriageScore() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(2); + alerts.get(0).put(METAALERT_FIELD, "meta_active_0"); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Load metaAlerts + List<Map<String, Object>> metaAlerts = buildMetaAlerts(1, MetaAlertStatus.ACTIVE, + Optional.of(Collections.singletonList(alerts.get(0)))); + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + addRecords(metaAlerts, getMetaAlertIndex(), METAALERT_TYPE); + + // Verify load was successful + List<GetRequest> createdDocs = metaAlerts.stream().map(metaAlert -> + new GetRequest((String) metaAlert.get(Constants.GUID), METAALERT_TYPE)) + .collect(Collectors.toList()); + createdDocs.addAll(alerts.stream().map(alert -> + new GetRequest((String) alert.get(Constants.GUID), SENSOR_NAME)) + .collect(Collectors.toList())); + findCreatedDocs(createdDocs); + + // Test descending + SortField sf = new SortField(); + sf.setField(getThreatTriageField()); + sf.setSortOrder(SortOrder.DESC.getSortOrder()); + SearchRequest sr = new SearchRequest(); + sr.setQuery("*:*"); + sr.setSize(5); + sr.setIndices(Arrays.asList(getTestIndexName(), METAALERT_TYPE)); + sr.setSort(Collections.singletonList(sf)); + + SearchResponse result = metaDao.search(sr); + List<SearchResult> results = result.getResults(); + Assert.assertEquals(2, results.size()); + Assert.assertEquals("meta_active_0", results.get((0)).getId()); + Assert.assertEquals("message_1", results.get((1)).getId()); + + // Test ascending + SortField sfAsc = new SortField(); + sfAsc.setField(getThreatTriageField()); + sfAsc.setSortOrder(SortOrder.ASC.getSortOrder()); + SearchRequest srAsc = new SearchRequest(); + srAsc.setQuery("*:*"); + srAsc.setSize(2); + srAsc.setIndices(Arrays.asList(getTestIndexName(), METAALERT_TYPE)); + srAsc.setSort(Collections.singletonList(sfAsc)); + result = metaDao.search(srAsc); + results = result.getResults(); + Assert.assertEquals("message_1", results.get((0)).getId()); + Assert.assertEquals("meta_active_0", results.get((1)).getId()); + Assert.assertEquals(2, results.size()); + } + + @Test public void getAllMetaAlertsForAlertShouldThrowExceptionForEmptyGuid() throws Exception { try { metaDao.getAllMetaAlertsForAlert(""); @@ -960,6 +1015,7 @@ public abstract class MetaAlertIntegrationTest { metaAlert.put(Constants.GUID, guid); metaAlert.put(getSourceTypeField(), METAALERT_TYPE); metaAlert.put(STATUS_FIELD, status.getStatusString()); + metaAlert.put(getThreatTriageField(), 100.0d); if (alerts.isPresent()) { List<Map<String, Object>> alertsList = alerts.get(); metaAlert.put(ALERT_FIELD, alertsList); http://git-wip-us.apache.org/repos/asf/metron/blob/28f4b570/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml index 63e729b..6555bf6 100644 --- a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml @@ -28,7 +28,6 @@ <field name="timestamp" type="plong" indexed="true" stored="true"/> <field name="score" type="pdouble" indexed="true" stored="true"/> <field name="status" type="string" indexed="true" stored="true"/> - <field name="threat:triage:score" type="pdouble" indexed="true" stored="true"/> <field name="average" type="pdouble" indexed="true" stored="true"/> <field name="min" type="pdouble" indexed="true" stored="true"/> <field name="median" type="pdouble" indexed="true" stored="true"/> @@ -40,6 +39,11 @@ <!-- Ensure that metaalerts child field is multivalued --> <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> + <!-- Threat Intel Scoring Field --> + <!-- This is a double from the method of calculation. It'll still sort alongside pfloat --> + <dynamicField name="*score" type="pdouble" multiValued="false" docValues="true"/> + + <!-- Catch all, if we don't know about it, it gets dropped. --> <dynamicField name="*" type="ignored" indexed="true" stored="true" multiValued="false" docValues="true"/> <uniqueKey>guid</uniqueKey>
