Repository: metron Updated Branches: refs/heads/feature/METRON-1416-upgrade-solr a7a41a55a -> cdbe7ac1f
METRON-1482 Update REST to work with Solr (merrimanr) closes apache/metron#957 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cdbe7ac1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cdbe7ac1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cdbe7ac1 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: cdbe7ac1f291c8d5be505973aaf01aa1d8a91920 Parents: a7a41a5 Author: merrimanr <[email protected]> Authored: Thu Mar 8 14:08:22 2018 -0600 Committer: merrimanr <[email protected]> Committed: Thu Mar 8 14:08:22 2018 -0600 ---------------------------------------------------------------------- .../CURRENT/package/scripts/rest_commands.py | 1 + .../metron-rest/src/main/scripts/metron-rest.sh | 29 +++-- .../metron/solr/dao/SolrMetaAlertDao.java | 126 +++++++++++++++++++ .../apache/metron/solr/dao/SolrSearchDao.java | 5 +- 4 files changed, 150 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 4f717bb..c3d3ec1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -101,6 +101,7 @@ class RestCommands: "export METRON_INDEX_CP={metron_indexing_classpath};" "export METRON_LOG_DIR={metron_log_dir};" "export METRON_PID_FILE={pid_file};" + "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};" "{metron_home}/bin/metron-rest.sh;" "unset METRON_JDBC_PASSWORD;" )) http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-interface/metron-rest/src/main/scripts/metron-rest.sh ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh index f9a2b69..e0eeaa4 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh @@ -84,19 +84,32 @@ if [ ${METRON_JDBC_CLIENT_PATH} ]; then METRON_REST_CLASSPATH+=":${METRON_JDBC_CLIENT_PATH}" fi -# Use a custom indexing jar if provided, else pull the metron-elasticsearch uber jar +# Use metron-elasticsearch uber jar by default +indexing_jar_pattern="${METRON_HOME}/lib/metron-elasticsearch*uber.jar" +# Use metron-solr uber jar if ra indexing writer set to Solr +if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then + indexing_jar_pattern="${METRON_HOME}/lib/metron-solr*uber.jar" +fi +# Use a custom indexing jar if provided if [ ${METRON_INDEX_CP} ]; then - echo "Default metron indexing jar is: ${METRON_INDEX_CP}" - METRON_REST_CLASSPATH+=":${METRON_INDEX_CP}" -else - indexing_jar_pattern="${METRON_HOME}/lib/metron-elasticsearch*uber.jar" - indexing_files=( ${indexing_jar_pattern} ) - echo "Default metron indexing jar is: ${indexing_files[0]}" - METRON_REST_CLASSPATH+=":${indexing_files[0]}" + indexing_jar_pattern="${METRON_INDEX_CP}" fi +indexing_files=( ${indexing_jar_pattern} ) +echo "Metron indexing jar is: ${indexing_files[0]}" +METRON_REST_CLASSPATH+=":${indexing_files[0]}" echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}" +#Use Solr daos if ra indexing writer set to Solr +if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then + METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao" + METRON_METAALERT_DAO=" --meta.dao.impl=org.apache.metron.solr.dao.SolrMetaAlertDao" + echo "METRON_INDEX_DAO=${METRON_INDEX_DAO}" + echo "METRON_METAALERT_DAO=${METRON_METAALERT_DAO}" + METRON_SPRING_OPTIONS+=${METRON_INDEX_DAO} + METRON_SPRING_OPTIONS+=${METRON_METAALERT_DAO} +fi + echo "Starting application" ${JAVA_HOME}/bin/java ${METRON_JVMFLAGS} \ -cp ${METRON_REST_CLASSPATH} \ http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java new file mode 100644 index 0000000..389cb4e --- /dev/null +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.dao; + +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.MetaAlertDao; +import org.apache.metron.indexing.dao.MultiIndexDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; +import org.apache.metron.indexing.dao.search.*; +import org.apache.metron.indexing.dao.update.Document; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class SolrMetaAlertDao implements MetaAlertDao { + + private SolrDao solrDao; + + @Override + public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException { + return null; + } + + @Override + public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) throws InvalidCreateException, IOException { + return null; + } + + @Override + public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> getRequests) throws IOException { + return false; + } + + @Override + public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> getRequests) throws IOException { + return false; + } + + @Override + public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) throws IOException { + return false; + } + + @Override + public void init(IndexDao indexDao) { + + } + + @Override + public void init(IndexDao indexDao, Optional<String> threatSort) { + if (indexDao instanceof MultiIndexDao) { + MultiIndexDao multiIndexDao = (MultiIndexDao) indexDao; + for (IndexDao childDao : multiIndexDao.getIndices()) { + if (childDao instanceof SolrDao) { + this.solrDao = (SolrDao) childDao; + } + } + } else if (indexDao instanceof SolrDao) { + this.solrDao = (SolrDao) indexDao; + } else { + throw new IllegalArgumentException( + "Need an SolrDao when using SolrMetaAlertDao" + ); + } + } + + @Override + public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { + return solrDao.search(searchRequest); + } + + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + return solrDao.group(groupRequest); + } + + @Override + public void init(AccessConfig config) { + + } + + @Override + public Document getLatest(String guid, String sensorType) throws IOException { + return solrDao.getLatest(guid, sensorType); + } + + @Override + public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException { + return solrDao.getAllLatest(getRequests); + } + + @Override + public void update(Document update, Optional<String> index) throws IOException { + solrDao.update(update, index); + } + + @Override + public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException { + solrDao.batchUpdate(updates); + } + + @Override + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { + return solrDao.getColumnMetadata(indices); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java index 378c976..2acf3c9 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java @@ -223,12 +223,11 @@ public class SolrSearchDao implements SearchDao { private SearchResult getSearchResult(SolrDocument solrDocument, Optional<List<String>> fields) { SearchResult searchResult = new SearchResult(); searchResult.setId((String) solrDocument.getFieldValue(Constants.GUID)); - Map<String, Object> source; + final Map<String, Object> source = new HashMap<>(); if (fields.isPresent()) { - source = new HashMap<>(); fields.get().forEach(field -> source.put(field, solrDocument.getFieldValue(field))); } else { - source = solrDocument.getFieldValueMap(); + solrDocument.getFieldNames().forEach(field -> source.put(field, solrDocument.getFieldValue(field))); } searchResult.setSource(source); return searchResult;
