Repository: metron Updated Branches: refs/heads/master 4116b9907 -> e25031905
METRON-1566 Alert updates are not propagated to metaalert child alerts (merrimanr) closes apache/metron#1018 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e2503190 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e2503190 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e2503190 Branch: refs/heads/master Commit: e25031905a3c85aeadd2063486abaa38c94a1eb4 Parents: 4116b99 Author: merrimanr <[email protected]> Authored: Wed May 23 08:57:19 2018 -0500 Committer: merrimanr <[email protected]> Committed: Wed May 23 08:57:19 2018 -0500 ---------------------------------------------------------------------- .../dao/ElasticsearchMetaAlertDao.java | 17 ++++++--- .../ElasticsearchMetaAlertIntegrationTest.java | 39 ++++++++++++++++++++ .../apache/metron/indexing/dao/HBaseDao.java | 2 +- .../integration/HBaseDaoIntegrationTest.java | 17 +++++++++ 4 files changed, 68 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/e2503190/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index 2c42b08..d12e40c 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -430,7 +430,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { // We need to update an alert itself. Only that portion of the update can be delegated. // We still need to get meta alerts potentially associated with it and update. Collection<Document> metaAlerts = getMetaAlertsForAlert(update.getGuid()).getResults().stream() - .map(searchResult -> new Document(searchResult.getSource(), searchResult.getId(), METAALERT_TYPE, 0L)) + .map(searchResult -> new Document(searchResult.getSource(), searchResult.getId(), METAALERT_TYPE, update.getTimestamp())) .collect(Collectors.toList()); // Each meta alert needs to be updated with the new alert for (Document metaAlert : metaAlerts) { @@ -468,12 +468,17 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { @Override public void patch(PatchRequest request, Optional<Long> timestamp) throws OriginalNotFoundException, IOException { - if (isPatchAllowed(request)) { - Document d = getPatchedDocument(request, timestamp); - indexDao.update(d, Optional.ofNullable(request.getIndex())); + if (METAALERT_TYPE.equals(request.getSensorType())) { + if (isPatchAllowed(request)) { + Document d = getPatchedDocument(request, timestamp); + indexDao.update(d, Optional.ofNullable(request.getIndex())); + } else { + throw new IllegalArgumentException("Meta alert patches are not allowed for /alert or /status paths. " + + "Please use the add/remove alert or update status functions instead."); + } } else { - throw new IllegalArgumentException("Meta alert patches are not allowed for /alert or /status paths. " - + "Please use the add/remove alert or update status functions instead."); + Document d = getPatchedDocument(request, timestamp); + update(d, Optional.ofNullable(request.getIndex())); } } http://git-wip-us.apache.org/repos/asf/metron/blob/e2503190/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 6c78883..5222a38 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -951,6 +951,45 @@ public class ElasticsearchMetaAlertIntegrationTest { } } } + //modify the same message and modify the new field with the patch method + { + Map<String, Object> message0 = new HashMap<String, Object>(alerts.get(0)) { + { + put(NEW_FIELD, "metron3"); + } + }; + String guid = "" + message0.get(Constants.GUID); + PatchRequest patchRequest = new PatchRequest(); + patchRequest.setGuid(guid); + patchRequest.setIndex(INDEX); + patchRequest.setSensorType(SENSOR_NAME); + patchRequest.setPatch(Collections.singletonList(new HashMap<String, Object>() {{ + put("op", "replace"); + put("path", "/" + NEW_FIELD); + put("value", "metron3"); + }})); + + metaDao.patch(patchRequest, Optional.empty()); + + { + // Verify ES is up-to-date + findUpdatedDoc(message0, guid, SENSOR_NAME); + long cnt = getMatchingAlertCount(NEW_FIELD, message0.get(NEW_FIELD)); + if (cnt == 0) { + Assert.fail("Elasticsearch alert not updated!"); + } + } + { + // Verify meta alerts in ES are up-to-date + long cnt = getMatchingMetaAlertCount(NEW_FIELD, "metron3"); + if (cnt == 0) { + Assert.fail("Active metaalert was not updated!"); + } + if (cnt != 1) { + Assert.fail("Elasticsearch metaalerts not updated correctly!"); + } + } + } } @Test http://git-wip-us.apache.org/repos/asf/metron/blob/e2503190/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java index ebb9907..7f37a9a 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java @@ -250,7 +250,7 @@ public class HBaseDao implements IndexDao { protected Put buildPut(Document update) throws IOException { Key k = new Key(update.getGuid(), update.getSensorType()); Put put = new Put(Key.toBytes(k)); - long ts = update.getTimestamp() == null ? System.currentTimeMillis() : update.getTimestamp(); + long ts = update.getTimestamp() == null || update.getTimestamp() == 0 ? System.currentTimeMillis() : update.getTimestamp(); byte[] columnQualifier = Bytes.toBytes(ts); byte[] doc = JSONUtils.INSTANCE.toJSONPretty(update.getDocument()); put.addColumn(cf, columnQualifier, doc); http://git-wip-us.apache.org/repos/asf/metron/blob/e2503190/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java index aa32aa0..f57a101 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java @@ -126,6 +126,23 @@ public class HBaseDaoIntegrationTest { } @Test + public void shouldGetLatestWithInvalidTimestamp() throws Exception { + // Load alert + Document alert = buildAlerts(1).get(0); + hbaseDao.update(alert, Optional.empty()); + + Document actualDocument = hbaseDao.getLatest("message_0", SENSOR_TYPE); + Assert.assertEquals(alert, actualDocument); + + alert.getDocument().put("field", "value"); + alert.setTimestamp(0L); + hbaseDao.update(alert, Optional.empty()); + + actualDocument = hbaseDao.getLatest("message_0", SENSOR_TYPE); + Assert.assertEquals(alert.getDocument(), actualDocument.getDocument()); + } + + @Test public void shouldGetAllLatest() throws Exception { // Load alerts List<Document> alerts = buildAlerts(15);
