[
https://issues.apache.org/jira/browse/METRON-1301?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16240775#comment-16240775
]
ASF GitHub Bot commented on METRON-1301:
----------------------------------------
Github user nickwallen commented on a diff in the pull request:
https://github.com/apache/metron/pull/832#discussion_r149182933
--- Diff:
metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
---
@@ -98,78 +121,153 @@ public ElasticsearchDao() {
//uninitialized.
}
- private static Map<String, FieldType> elasticsearchSearchTypeMap;
-
- static {
- Map<String, FieldType> fieldTypeMap = new HashMap<>();
- fieldTypeMap.put("string", FieldType.STRING);
- fieldTypeMap.put("ip", FieldType.IP);
- fieldTypeMap.put("integer", FieldType.INTEGER);
- fieldTypeMap.put("long", FieldType.LONG);
- fieldTypeMap.put("date", FieldType.DATE);
- fieldTypeMap.put("float", FieldType.FLOAT);
- fieldTypeMap.put("double", FieldType.DOUBLE);
- fieldTypeMap.put("boolean", FieldType.BOOLEAN);
- elasticsearchSearchTypeMap = Collections.unmodifiableMap(fieldTypeMap);
- }
-
@Override
public SearchResponse search(SearchRequest searchRequest) throws
InvalidSearchException {
return search(searchRequest, new
QueryStringQueryBuilder(searchRequest.getQuery()));
}
/**
* Defers to a provided {@link
org.elasticsearch.index.query.QueryBuilder} for the query.
- * @param searchRequest The request defining the parameters of the search
+ * @param request The request defining the parameters of the search
* @param queryBuilder The actual query to be run. Intended for if the
SearchRequest requires wrapping
* @return The results of the query
* @throws InvalidSearchException When the query is malformed or the
current state doesn't allow search
*/
- protected SearchResponse search(SearchRequest searchRequest,
QueryBuilder queryBuilder) throws InvalidSearchException {
+ protected SearchResponse search(SearchRequest request, QueryBuilder
queryBuilder) throws InvalidSearchException {
+ org.elasticsearch.action.search.SearchRequest esRequest;
+ org.elasticsearch.action.search.SearchResponse esResponse;
+
if(client == null) {
throw new InvalidSearchException("Uninitialized Dao! You must call
init() prior to use.");
}
- if (searchRequest.getSize() > accessConfig.getMaxSearchResults()) {
+
+ if (request.getSize() > accessConfig.getMaxSearchResults()) {
throw new InvalidSearchException("Search result size must be less
than " + accessConfig.getMaxSearchResults());
}
- SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
+
+ esRequest = buildSearchRequest(request, queryBuilder);
+ esResponse = searchSubmitter.submitSearch(esRequest);
+ return buildSearchResponse(request, esResponse);
+ }
+
+ /**
+ * Builds an Elasticsearch search request.
+ * @param searchRequest The Metron search request.
+ * @param queryBuilder
+ * @return An Elasticsearch search request.
+ */
+ private org.elasticsearch.action.search.SearchRequest buildSearchRequest(
+ SearchRequest searchRequest,
+ QueryBuilder queryBuilder) throws InvalidSearchException {
+
+ LOG.debug("Got search request; request={}",
ElasticsearchUtils.toJSON(searchRequest));
+ SearchSourceBuilder searchBuilder = new SearchSourceBuilder()
.size(searchRequest.getSize())
.from(searchRequest.getFrom())
.query(queryBuilder)
.trackScores(true);
- searchRequest.getSort().forEach(sortField ->
searchSourceBuilder.sort(sortField.getField(),
getElasticsearchSortOrder(sortField.getSortOrder())));
- Optional<List<String>> fields = searchRequest.getFields();
- if (fields.isPresent()) {
- searchSourceBuilder.fields(fields.get());
- } else {
- searchSourceBuilder.fetchSource(true);
+
+ // column metadata needed to understand the type of each sort field
+ Map<String, Map<String, FieldType>> meta;
+ try {
+ meta = getColumnMetadata(searchRequest.getIndices());
+ } catch(IOException e) {
+ throw new InvalidSearchException("Unable to get column metadata", e);
}
- Optional<List<String>> facetFields = searchRequest.getFacetFields();
- if (facetFields.isPresent()) {
- facetFields.get().forEach(field ->
searchSourceBuilder.aggregation(new
TermsBuilder(getFacentAggregationName(field)).field(field)));
+
+ // handle sort fields
+ for(SortField sortField : searchRequest.getSort()) {
+
+ // what type is the sort field?
+ FieldType sortFieldType = meta
+ .values()
+ .stream()
+ .filter(e -> e.containsKey(sortField.getField()))
+ .map(m -> m.get(sortField.getField()))
+ .findFirst()
+ .orElse(FieldType.OTHER);
+
+ // sort order - if ASC, then missing values sorted last. Otherwise,
missing values sorted first
+ org.elasticsearch.search.sort.SortOrder sortOrder =
getElasticsearchSortOrder(sortField.getSortOrder());
+ String missingSortOrder;
+ if(sortOrder == org.elasticsearch.search.sort.SortOrder.DESC) {
+ missingSortOrder = SORT_MISSING_LAST;
+ } else {
+ missingSortOrder = SORT_MISSING_FIRST;
+ }
--- End diff --
Here is the additional logic to handle sorting of 'missing' fields.
> Alerts UI - Sorting on Triage Score Unexpectedly Filters Some Records
> ---------------------------------------------------------------------
>
> Key: METRON-1301
> URL: https://issues.apache.org/jira/browse/METRON-1301
> Project: Metron
> Issue Type: Bug
> Affects Versions: 0.4.1
> Reporter: Nick Allen
> Assignee: Nick Allen
> Fix For: Next + 1
>
> Attachments: 01-Alerts-UI-default-view.png, 02-Sort-on-Score-field.png
>
>
> Sorting on a field like threat triage score in the Alerts UI removes any
> records that do not have a threat triage score defined from the search
> results.
> For example, I have 7 records when sorted by timestamp. All 7 records have a
> timestamp field.
> After sorting by score (threat triage score) there are only 5 records. The 2
> records missing a threat triage score are no longer included.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
