Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a89a72c3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a89a72c3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a89a72c3 Branch: refs/heads/master Commit: a89a72c3b2ceb5066fd7a310636507ce274c66ad Parents: 10271c4 b081e80 Author: cstella <ceste...@gmail.com> Authored: Fri Jun 15 20:32:18 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Fri Jun 15 20:32:18 2018 -0400 ---------------------------------------------------------------------- .../alert-details/alert-details.component.html | 4 +-- .../alert-details/alert-details.component.ts | 24 ++++++++++++++--- .../alerts-list/alerts-list.component.html | 4 +-- .../alerts/alerts-list/alerts-list.component.ts | 13 ++++++---- .../src/app/alerts/alerts-list/query-builder.ts | 5 ++-- .../table-view/table-view.component.html | 18 ++++++------- .../table-view/table-view.component.ts | 19 ++++++++++++-- .../tree-view/tree-view.component.html | 16 ++++++------ .../tree-view/tree-view.component.ts | 27 +++++++++++++++++--- .../meta-alerts/meta-alerts.component.html | 2 +- .../alerts/meta-alerts/meta-alerts.component.ts | 11 +++++--- .../src/app/model/group-request.ts | 2 +- .../src/app/service/global-config.service.ts | 16 +++++++++--- .../metron-alerts/src/app/utils/constants.ts | 3 +-- .../ElasticsearchMetaAlertIntegrationTest.java | 5 ++-- 15 files changed, 120 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts ---------------------------------------------------------------------- diff --cc metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts index ffd7dd0,c939f04..39d47ac --- a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts +++ b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts @@@ -29,8 -30,8 +30,9 @@@ import {Patch} from '../../model/patch' import {AlertComment} from './alert-comment'; import {AuthenticationService} from '../../service/authentication.service'; import {MetronDialogBox} from '../../shared/metron-dialog-box'; - import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants'; -import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../../utils/constants'; +import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request"; ++import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants'; + import { GlobalConfigService } from '../../service/global-config.service'; export enum AlertState { NEW, OPEN, ESCALATE, DISMISS, RESOLVE http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts ---------------------------------------------------------------------- diff --cc metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts index 98459db,4496e37..07f7015 --- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts +++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts @@@ -36,10 -36,10 +36,10 @@@ import {AlertSearchDirective} from '../ import {SearchResponse} from '../../model/search-response'; import {ElasticsearchUtils} from '../../utils/elasticsearch-utils'; import {Filter} from '../../model/filter'; - import {THREAT_SCORE_FIELD_NAME, TIMESTAMP_FIELD_NAME, ALL_TIME} from '../../utils/constants'; + import {TIMESTAMP_FIELD_NAME, ALL_TIME} from '../../utils/constants'; import {TableViewComponent} from './table-view/table-view.component'; import {Pagination} from '../../model/pagination'; -import {META_ALERTS_SENSOR_TYPE, META_ALERTS_INDEX} from '../../utils/constants'; +import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants'; import {MetaAlertService} from '../../service/meta-alert.service'; import {Facets} from '../../model/facets'; import { GlobalConfigService } from '../../service/global-config.service'; http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.html ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.ts ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/utils/constants.ts ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 224ec56,933fa2a..c05efc1 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@@ -38,24 -41,36 +38,25 @@@ import java.util.HashMap import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; + + import com.google.common.collect.ImmutableList; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; - import org.apache.metron.guava.collect.ImmutableList; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.MetaAlertDao; -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertIntegrationTest; import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; import org.apache.metron.indexing.dao.search.GetRequest; -import org.apache.metron.indexing.dao.search.Group; -import org.apache.metron.indexing.dao.search.GroupRequest; -import org.apache.metron.indexing.dao.search.GroupResponse; -import org.apache.metron.indexing.dao.search.GroupResult; -import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; -import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.search.SortField; -import org.apache.metron.indexing.dao.update.Document; -import org.apache.metron.indexing.dao.update.OriginalNotFoundException; -import org.apache.metron.indexing.dao.update.PatchRequest; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@@ -291,87 -1136,131 +292,87 @@@ public class ElasticsearchMetaAlertInte return cnt; } - protected long getMatchingMetaAlertCount(String fieldName, String fieldValue) throws IOException, InterruptedException { + @Override + protected long getMatchingMetaAlertCount(String fieldName, String fieldValue) + throws IOException, InterruptedException { long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { - List<Map<String, Object>> docs = es.getAllIndexedDocs(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC); + List<Map<String, Object>> docs = es + .getAllIndexedDocs(METAALERTS_INDEX, METAALERT_DOC); cnt = docs - .stream() - .filter(d -> { - List<Map<String, Object>> alerts = (List<Map<String, Object>>) d - .get(MetaAlertDao.ALERT_FIELD); + .stream() + .filter(d -> { + @SuppressWarnings("unchecked") + List<Map<String, Object>> alerts = (List<Map<String, Object>>) d + .get(ALERT_FIELD); + + for (Map<String, Object> alert : alerts) { + Object newField = alert.get(fieldName); + if (newField != null && newField.equals(fieldValue)) { + return true; + } + } + + return false; + }).count(); + } + return cnt; + } - for (Map<String, Object> alert : alerts) { - Object newField = alert.get(fieldName); - if (newField != null && newField.equals(fieldValue)) { - return true; + @Override + protected void addRecords(List<Map<String, Object>> inputData, String index, String docType) + throws IOException { + es.add(index, docType, inputData.stream().map(m -> { + try { + return JSONUtils.INSTANCE.toJSON(m, true); + } catch (JsonProcessingException e) { + throw new IllegalStateException(e.getMessage(), e); } } - - return false; - }).count(); - } - return cnt; + ).collect(Collectors.toList()) + ); } - protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) - throws InterruptedException, IOException, OriginalNotFoundException { - for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { - Document doc = metaDao.getLatest(guid, sensorType); - if (doc != null && compareDocs(message0, doc.getDocument())) { - return; - } - } - throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES + " tries"); + @Override + protected void setupTypings() { + ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX) + .setType("test_doc") + .setSource(nestedAlertMapping) + .get(); } - private boolean compareDocs(Map<String, Object> expected, Map<String, Object> actual) { - if (expected.size() != actual.size()) { - return false; - } - for(String key: expected.keySet()) { - if (ALERT_FIELD.equals(key)) { - List<Map<String, Object>> expectedAlerts = (List<Map<String, Object>>) expected.get(MetaAlertDao.ALERT_FIELD); - ArrayList<Map<String, Object>> actualAlerts = (ArrayList<Map<String, Object>>) actual.get(MetaAlertDao.ALERT_FIELD); - if (!expectedAlerts.containsAll(actualAlerts) || !actualAlerts.containsAll(expectedAlerts)) { - return false; - } - } else if (!expected.get(key).equals(actual.get(key))){ - return false; - } - } - return true; + @Override + protected String getTestIndexName() { + return INDEX_RAW; } - protected boolean findCreatedDoc(String guid, String sensorType) - throws InterruptedException, IOException, OriginalNotFoundException { - for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { - Document doc = metaDao.getLatest(guid, sensorType); - if (doc != null) { - return true; - } - } - throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES + "tries"); + @Override + protected String getTestIndexFullName() { + return INDEX; } - protected boolean findCreatedDocs(List<GetRequest> getRequests) - throws InterruptedException, IOException, OriginalNotFoundException { - for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { - Iterable<Document> docs = metaDao.getAllLatest(getRequests); - if (docs != null) { - int docCount = 0; - for (Document doc: docs) { - docCount++; - } - if (getRequests.size() == docCount) { - return true; - } - } - } - throw new OriginalNotFoundException("Count not find guids after " + MAX_RETRIES + "tries"); + @Override + protected String getMetaAlertIndex() { + return METAALERTS_INDEX; } - protected List<Map<String, Object>> buildAlerts(int count) { - List<Map<String, Object>> inputData = new ArrayList<>(); - for (int i = 0; i < count; ++i) { - final String guid = "message_" + i; - Map<String, Object> alerts = new HashMap<>(); - alerts.put(Constants.GUID, guid); - alerts.put("source:type", SENSOR_NAME); - alerts.put(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, i); - alerts.put("timestamp", System.currentTimeMillis()); - inputData.add(alerts); - } - return inputData; + @Override + protected String getSourceTypeField() { + return ElasticsearchMetaAlertDao.SOURCE_TYPE_FIELD; } - protected List<Map<String, Object>> buildMetaAlerts(int count, MetaAlertStatus status, Optional<List<Map<String, Object>>> alerts) { - List<Map<String, Object>> inputData = new ArrayList<>(); - for (int i = 0; i < count; ++i) { - final String guid = "meta_" + status.getStatusString() + "_" + i; - inputData.add(buildMetaAlert(guid, status, alerts)); - } - return inputData; + @Override + protected void setEmptiedMetaAlertField(Map<String, Object> docMap) { + docMap.put(METAALERT_FIELD, new ArrayList<>()); } - protected Map<String, Object> buildMetaAlert(String guid, MetaAlertStatus status, Optional<List<Map<String, Object>>> alerts) { - Map<String, Object> metaAlert = new HashMap<>(); - metaAlert.put(Constants.GUID, guid); - metaAlert.put("source:type", METAALERT_TYPE); - metaAlert.put(MetaAlertDao.STATUS_FIELD, status.getStatusString()); - if (alerts.isPresent()) { - List<Map<String, Object>> alertsList = alerts.get(); - metaAlert.put(ALERT_FIELD, alertsList); - } - return metaAlert; + @Override + protected boolean isFiniteDoubleOnly() { + return true; } - protected void elasticsearchAdd(List<Map<String, Object>> inputData, String index, String docType) - throws IOException { - es.add(index, docType, inputData.stream().map(m -> { - try { - return JSONUtils.INSTANCE.toJSON(m, true); - } catch (JsonProcessingException e) { - throw new IllegalStateException(e.getMessage(), e); - } - } - ).collect(Collectors.toList()) - ); + @Override + protected boolean isEmptyMetaAlertList() { + return true; } - } + }
