AMBARI-19468. Run ZK operations in logsearch internally and enable ACL security (oleewere)
Change-Id: Ida845767ee34c4f58e9b791cd6b131e22e0439f6 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/548a0b6a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/548a0b6a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/548a0b6a Branch: refs/heads/branch-2.5 Commit: 548a0b6a31c486a6f24c0d431be0d1a247b72250 Parents: e382df2 Author: oleewere <[email protected]> Authored: Wed Jan 11 16:20:44 2017 +0100 Committer: oleewere <[email protected]> Committed: Thu Jan 19 14:13:04 2017 +0100 ---------------------------------------------------------------------- .../libraries/functions/solr_cloud_util.py | 22 +- .../ambari-logsearch-portal/pom.xml | 13 +- .../configsets/audit_logs/conf/managed-schema | 100 +++--- .../logsearch/common/ACLPropertiesSplitter.java | 70 +++++ .../ambari/logsearch/common/MessageEnums.java | 3 + .../ambari/logsearch/conf/SecurityConfig.java | 57 +++- .../logsearch/conf/SolrAuditLogPropsConfig.java | 27 ++ .../ambari/logsearch/conf/SolrConfig.java | 71 ++--- .../conf/SolrConnectionPropsConfig.java | 29 ++ .../ambari/logsearch/conf/SolrPropsConfig.java | 12 + .../conf/global/SolrAuditLogsState.java | 68 ++++ .../conf/global/SolrCollectionState.java | 33 ++ .../conf/global/SolrServiceLogsState.java | 59 ++++ .../conf/global/SolrUserConfigState.java | 60 ++++ .../configurer/LogfeederFilterConfigurer.java | 66 ++++ .../configurer/SolrAuditAliasConfigurer.java | 136 ++++++++ .../configurer/SolrCollectionConfigurer.java | 230 ++++++++++++++ .../logsearch/configurer/SolrConfigurer.java | 23 ++ .../ambari/logsearch/dao/AuditSolrDao.java | 38 ++- .../logsearch/dao/ServiceLogsSolrDao.java | 30 +- .../ambari/logsearch/dao/SolrAliasDao.java | 121 ------- .../ambari/logsearch/dao/SolrCollectionDao.java | 313 ------------------- .../ambari/logsearch/dao/SolrDaoBase.java | 18 +- .../logsearch/dao/SolrSchemaFieldDao.java | 38 +-- .../ambari/logsearch/dao/UserConfigSolrDao.java | 39 ++- .../ambari/logsearch/doc/DocConstants.java | 7 + .../ambari/logsearch/handler/ACLHandler.java | 97 ++++++ .../handler/CreateCollectionHandler.java | 222 +++++++++++++ .../handler/ListCollectionHandler.java | 51 +++ .../handler/ReloadCollectionHandler.java | 45 +++ .../logsearch/handler/SolrZkRequestHandler.java | 26 ++ .../handler/UploadConfigurationHandler.java | 100 ++++++ .../ambari/logsearch/rest/StatusResource.java | 91 ++++++ .../ambari/logsearch/util/RESTErrorUtil.java | 10 +- .../AbstractLogsearchGlobalStateFilter.java | 100 ++++++ .../filters/LogsearchAuditLogsStateFilter.java | 51 +++ .../LogsearchServiceLogsStateFilter.java | 51 +++ .../filters/LogsearchUserConfigStateFilter.java | 52 +++ .../src/main/webapp/scripts/utils/Utils.js | 9 +- .../logsearch/solr/AmbariSolrCloudClient.java | 14 - .../solr/commands/CreateSaslUsersZkCommand.java | 60 ---- .../solr/commands/SecureSolrZNodeZkCommand.java | 17 +- .../ambari/logsearch/solr/util/AclUtils.java | 30 -- ambari-logsearch/docker/bin/start.sh | 6 - .../logsearch/logsearch-https.properties | 3 + .../test-config/logsearch/logsearch.properties | 3 + .../0.1.0/package/scripts/infra_solr.py | 13 + .../0.1.0/package/scripts/params.py | 12 + .../0.1.0/package/scripts/setup_infra_solr.py | 9 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 20 +- .../0.5.0/package/scripts/service_check.py | 2 +- .../0.5.0/package/scripts/setup_logsearch.py | 38 --- .../stacks/2.3/ATLAS/test_metadata_server.py | 6 +- .../stacks/2.4/LOGSEARCH/test_logsearch.py | 33 -- .../stacks/2.5/ATLAS/test_atlas_server.py | 3 +- .../stacks/2.5/RANGER/test_ranger_admin.py | 4 +- .../2.5/configs/ranger-admin-default.json | 2 +- .../2.5/configs/ranger-admin-secured.json | 2 +- .../stacks/2.6/RANGER/test_ranger_admin.py | 4 +- .../2.6/configs/ranger-admin-default.json | 2 +- .../2.6/configs/ranger-admin-secured.json | 2 +- 61 files changed, 2052 insertions(+), 821 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py index 2e48c6a..3e076b8 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py @@ -18,13 +18,14 @@ limitations under the License. """ import random from ambari_commons.constants import AMBARI_SUDO_BINARY +from ambari_jinja2 import Environment as JinjaEnvironment from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.core.resources.system import Directory, Execute, File -from resource_management.core.source import StaticFile, InlineTemplate +from resource_management.core.source import StaticFile __all__ = ["upload_configuration_to_zk", "create_collection", "setup_kerberos", "set_cluster_prop", - "setup_kerberos_plugin", "create_znode", "check_znode", "create_sasl_users"] + "setup_kerberos_plugin", "create_znode", "check_znode", "secure_solr_znode", "secure_znode"] def __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, separated_znode=False): sudo = AMBARI_SUDO_BINARY @@ -172,12 +173,12 @@ def secure_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_user Execute(secure_znode_cmd) -def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users=[]): +def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users_str=''): """ Secure solr znode - setup acls to 'cdrwa' for solr user, set 'r' only for the world, skipping /znode/configs and znode/collections (set those to 'cr' for the world) + sasl_users_str: comma separated sasl users """ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True) - sasl_users_str = ",".join(str(x) for x in sasl_users) secure_solr_znode_cmd = format('{solr_cli_prefix} --secure-solr-znode --jaas-file {jaas_file} --sasl-users {sasl_users_str}') Execute(secure_solr_znode_cmd) @@ -218,8 +219,19 @@ def setup_solr_client(config, custom_log4j = True, custom_log_location = None, l if custom_log4j: # use custom log4j content only, when infra is not installed on the cluster solr_client_log4j_content = config['configurations']['infra-solr-client-log4j']['content'] if log4jcontent is None else log4jcontent + context = { + 'solr_client_log': solr_client_log, + 'solr_client_log_maxfilesize': solr_client_log_maxfilesize, + 'solr_client_log_maxbackupindex': solr_client_log_maxbackupindex + } + template = JinjaEnvironment( + line_statement_prefix='%', + variable_start_string="{{", + variable_end_string="}}")\ + .from_string(solr_client_log4j_content) + File(format("{solr_client_dir}/log4j.properties"), - content=InlineTemplate(solr_client_log4j_content), + content=template.render(context), mode=0644 ) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index 9d17fe4..4bba464 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -642,11 +642,6 @@ </dependency> <dependency> <groupId>org.apache.ambari</groupId> - <artifactId>ambari-logsearch-solr-client</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ambari</groupId> <artifactId>ambari-metrics-common</artifactId> <version>${project.version}</version> </dependency> @@ -656,6 +651,14 @@ <version>1.3.1</version> </dependency> <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> <groupId>org.springframework.security.kerberos</groupId> <artifactId>spring-security-kerberos-core</artifactId> <version>1.0.1.RELEASE</version> http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema index 4cd412b..3f7fe3d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema @@ -15,30 +15,37 @@ See the License for the specific language governing permissions and limitations under the License. --> +<!-- Solr managed schema - automatically generated - DO NOT EDIT --> <schema name="audit-logs-schema" version="1.5"> <uniqueKey>id</uniqueKey> <fieldType name="binary" class="solr.BinaryField"/> <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> - <fieldType name="booleans" class="solr.BoolField" multiValued="true" sortMissingLast="true"/> - <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/> - <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/> - <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/> - <fieldType name="ignored" class="solr.StrField" multiValued="true" indexed="false" stored="false"/> - <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/> - <fieldType name="key_lower_case" class="solr.TextField" multiValued="false" sortMissingLast="true" omitNorms="true"> + <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/> + <fieldType name="date" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="ignored" class="solr.StrField" indexed="false" stored="false" multiValued="true"/> + <fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="key_lower_case" class="solr.TextField" omitNorms="true" sortMissingLast="true" multiValued="false"> <analyzer> <tokenizer class="solr.KeywordTokenizerFactory"/> <filter class="solr.LowerCaseFilterFactory"/> </analyzer> </fieldType> - <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="n_gram" class="solr.TextField" omitNorms="true" sortMissingLast="true"> + <analyzer> + <tokenizer class="solr.NGramTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> <fieldType name="random" class="solr.RandomSortField" indexed="true"/> <fieldType name="string" class="solr.StrField" sortMissingLast="true"/> - <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/> - <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" multiValued="true" positionIncrementGap="0"/> - <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/> - <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> - <fieldType name="text_std_token_lower_case" class="solr.TextField" multiValued="true" positionIncrementGap="100"> + <fieldType name="tdate" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="6"/> + <fieldType name="tdates" class="solr.TrieDateField" positionIncrementGap="0" multiValued="true" precisionStep="6"/> + <fieldType name="tdouble" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tdoubles" class="solr.TrieDoubleField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> + <fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100" multiValued="true"> <analyzer> <tokenizer class="solr.StandardTokenizerFactory"/> <filter class="solr.LowerCaseFilterFactory"/> @@ -50,45 +57,54 @@ <filter class="solr.LowerCaseFilterFactory"/> </analyzer> </fieldType> - <fieldType name="n_gram" class="solr.TextField" sortMissingLast="true" omitNorms="true"> - <analyzer> - <tokenizer class="solr.NGramTokenizerFactory"/> - <filter class="solr.LowerCaseFilterFactory"/> - </analyzer> - </fieldType> - - <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/> - <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> - <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/> - <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> - <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/> - <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> + <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tfloats" class="solr.TrieFloatField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> + <fieldType name="tint" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tints" class="solr.TrieIntField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> + <fieldType name="tlong" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tlongs" class="solr.TrieLongField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> <field name="_expire_at_" type="tdate" multiValued="false" stored="true"/> + <field name="_router_field_" type="int" multiValued="false" indexed="false" stored="false"/> <field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/> <field name="_version_" type="long" indexed="true" stored="true"/> - <field name="_router_field_" type="int" indexed="false" stored="false" multiValued="false"/> - <field name="access" type="key_lower_case" multiValued="false"/> <field name="action" type="key_lower_case" multiValued="false"/> <field name="agent" type="key_lower_case" multiValued="false"/> <field name="agentHost" type="key_lower_case" multiValued="false"/> + <field name="authType" type="key_lower_case"/> + <field name="bundle_id" type="key_lower_case" multiValued="false"/> + <field name="case_id" type="key_lower_case" multiValued="false"/> <field name="cliIP" type="key_lower_case" multiValued="false"/> <field name="cliType" type="key_lower_case" multiValued="false"/> <field name="cluster" type="key_lower_case" multiValued="false"/> - <field name="reqContext" type="key_lower_case" multiValued="true"/> + <field name="dst" type="key_lower_case"/> <field name="enforcer" type="key_lower_case" multiValued="false"/> - <field name="event_count" type="tlong" multiValued="false" docValues="true" default="1"/> + <field name="event_count" type="tlong" default="1" docValues="true" multiValued="false"/> <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/> - <field name="evtTime" type="tdate" docValues="true"/> + <field name="event_md5" type="string" multiValued="false"/> + <field name="evtTime" type="tdate" docValues="true"/> + <field name="file" type="key_lower_case" multiValued="false"/> + <field name="host" type="key_lower_case"/> <field name="id" type="string" multiValued="false" indexed="true" required="true" stored="true"/> + <field name="ip" type="key_lower_case"/> + <field name="level" type="key_lower_case"/> <field name="logType" type="key_lower_case" multiValued="false"/> + <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/> + <field name="logfile_line_number" type="tint" omitNorms="false"/> + <field name="logger_name" type="key_lower_case"/> + <field name="message_md5" type="string" multiValued="false"/> + <field name="path" type="key_lower_case"/> + <field name="perm" type="key_lower_case"/> <field name="policy" type="tlong" docValues="true"/> <field name="proxyUsers" type="key_lower_case" multiValued="true"/> <field name="reason" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/> <field name="repo" type="key_lower_case" multiValued="false"/> <field name="repoType" type="tint" multiValued="false" docValues="true"/> + <field name="reqContext" type="key_lower_case" multiValued="true"/> <field name="reqData" type="text_std_token_lower_case" multiValued="false"/> <field name="reqUser" type="key_lower_case" multiValued="false"/> + <field name="req_caller_id" type="key_lower_case" multiValued="false"/> + <field name="req_self_id" type="key_lower_case" multiValued="false"/> <field name="resType" type="key_lower_case" multiValued="false"/> <field name="resource" type="key_lower_case" multiValued="false"/> <field name="result" type="tint" multiValued="false"/> @@ -96,22 +112,14 @@ <field name="sess" type="key_lower_case" multiValued="false"/> <field name="tags" type="key_lower_case" multiValued="true"/> <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/> + <field name="task_id" type="tlong"/> <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/> - - <field name="bundle_id" type="key_lower_case" multiValued="false"/> - <field name="case_id" type="key_lower_case" multiValued="false"/> - <field name="req_caller_id" type="key_lower_case" multiValued="false"/> - <field name="req_self_id" type="key_lower_case" multiValued="false"/> - <field name="event_md5" type="string" multiValued="false"/> - <field name="file" type="key_lower_case" multiValued="false"/> - <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/> - <field name="logfile_line_number" type="tint" omitNorms="false"/> - <field name="message_md5" type="string" multiValued="false"/> <field name="type" type="key_lower_case" multiValued="false"/> - - <dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="true"/> - <dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="true"/> - <dynamicField name='key_*' type="key_lower_case" multiValued="false" stored="true"/> - <dynamicField name="ws_*" type="text_ws" multiValued="false" omitNorms="false" stored="true"/> + <field name="ugi" type="key_lower_case"/> + + <dynamicField name="ngram_*" type="n_gram" multiValued="false" stored="true"/> + <dynamicField name="std_*" type="text_std_token_lower_case" multiValued="false" stored="true"/> + <dynamicField name="key_*" type="key_lower_case" multiValued="false" stored="true"/> + <dynamicField name="ws_*" type="text_ws" omitNorms="false" multiValued="false" stored="true"/> </schema> http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java new file mode 100644 index 0000000..35a2b1b --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.common; + +import com.google.common.base.Splitter; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; + +import javax.inject.Named; +import java.util.ArrayList; +import java.util.List; + +@Named +public class ACLPropertiesSplitter { + + public List<ACL> parseAcls(String aclStr) { + List<ACL> acls = new ArrayList<>(); + List<String> aclStrList = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(aclStr); + for (String unparcedAcl : aclStrList) { + String[] parts = unparcedAcl.split(":"); + if (parts.length == 3) { + acls.add(new ACL(parsePermission(parts[2]), new Id(parts[0], parts[1]))); + } + } + return acls; + } + + private Integer parsePermission(String permission) { + int permissionCode = 0; + for (char each : permission.toLowerCase().toCharArray()) { + switch (each) { + case 'r': + permissionCode |= ZooDefs.Perms.READ; + break; + case 'w': + permissionCode |= ZooDefs.Perms.WRITE; + break; + case 'c': + permissionCode |= ZooDefs.Perms.CREATE; + break; + case 'd': + permissionCode |= ZooDefs.Perms.DELETE; + break; + case 'a': + permissionCode |= ZooDefs.Perms.ADMIN; + break; + default: + throw new IllegalArgumentException("Unsupported permission: " + permission); + } + } + return permissionCode; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java index c0a7f32..4f1725f 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java @@ -30,6 +30,9 @@ public enum MessageEnums { ERROR_DUPLICATE_OBJECT("logsearch.error.duplicate_object", "Error creating duplicate object"), ERROR_SYSTEM("logsearch.error.system", "System Error. Please try later."), SOLR_ERROR("logsearch.solr.error","Something went wrong, For more details check the logs or configuration."), + ZNODE_NOT_READY("logsearch.zk.znode.error", "ZNode is not available."), + ZK_CONFIG_NOT_READY("logsearch.zk.config.error", "Collection configuration has not uploaded yet"), + SOLR_COLLECTION_NOT_READY("logsearch.solr.collection.error", "Solr has not accessible yet for collection."), // Common Validations INVALID_PASSWORD("logsearch.validation.invalid_password", "Invalid password"), http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java index c99a738..115dcc3 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java @@ -19,13 +19,17 @@ package org.apache.ambari.logsearch.conf; import com.google.common.collect.Lists; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthFailureHandler; import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthSuccessHandler; import org.apache.ambari.logsearch.web.authenticate.LogsearchLogoutSuccessHandler; +import org.apache.ambari.logsearch.web.filters.LogsearchAuditLogsStateFilter; import org.apache.ambari.logsearch.web.filters.LogsearchAuthenticationEntryPoint; import org.apache.ambari.logsearch.web.filters.LogsearchKRBAuthenticationFilter; import org.apache.ambari.logsearch.web.filters.LogsearchJWTFilter; import org.apache.ambari.logsearch.web.filters.LogsearchSecurityContextFormationFilter; +import org.apache.ambari.logsearch.web.filters.LogsearchServiceLogsStateFilter; +import org.apache.ambari.logsearch.web.filters.LogsearchUserConfigStateFilter; import org.apache.ambari.logsearch.web.filters.LogsearchUsernamePasswordAuthenticationFilter; import org.apache.ambari.logsearch.web.security.LogsearchAuthenticationProvider; import org.springframework.context.annotation.Bean; @@ -37,11 +41,11 @@ import org.springframework.security.config.http.SessionCreationPolicy; import org.springframework.security.web.access.intercept.FilterSecurityInterceptor; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; -import org.springframework.security.web.util.matcher.NegatedRequestMatcher; import org.springframework.security.web.util.matcher.OrRequestMatcher; import org.springframework.security.web.util.matcher.RequestMatcher; import javax.inject.Inject; +import javax.inject.Named; import java.util.List; import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_SESSION_ID; @@ -53,6 +57,27 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { @Inject private AuthPropsConfig authPropsConfig; + @Inject + private SolrServiceLogPropsConfig solrServiceLogPropsConfig; + + @Inject + private SolrAuditLogPropsConfig solrAuditLogPropsConfig; + + @Inject + private SolrUserPropsConfig solrUserPropsConfig; + + @Inject + @Named("solrServiceLogsState") + private SolrCollectionState solrServiceLogsState; + + @Inject + @Named("solrAuditLogsState") + private SolrCollectionState solrAuditLogsState; + + @Inject + @Named("solrUserConfigState") + private SolrCollectionState solrUserConfigState; + @Override protected void configure(HttpSecurity http) throws Exception { http @@ -76,6 +101,9 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { .addFilterBefore(logsearchUsernamePasswordAuthenticationFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(logsearchKRBAuthenticationFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterAfter(securityContextFormationFilter(), FilterSecurityInterceptor.class) + .addFilterAfter(logsearchUserConfigFilter(), LogsearchSecurityContextFormationFilter.class) + .addFilterAfter(logsearchAuditLogFilter(), LogsearchSecurityContextFormationFilter.class) + .addFilterAfter(logsearchServiceLogFilter(), LogsearchSecurityContextFormationFilter.class) .addFilterBefore(logsearchJwtFilter(), LogsearchSecurityContextFormationFilter.class) .logout() .logoutUrl("/logout.html") @@ -124,6 +152,21 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { } @Bean + public LogsearchServiceLogsStateFilter logsearchServiceLogFilter() { + return new LogsearchServiceLogsStateFilter(serviceLogsRequestMatcher(), solrServiceLogsState, solrServiceLogPropsConfig); + } + + @Bean + public LogsearchAuditLogsStateFilter logsearchAuditLogFilter() { + return new LogsearchAuditLogsStateFilter(auditLogsRequestMatcher(), solrAuditLogsState, solrAuditLogPropsConfig); + } + + @Bean + public LogsearchUserConfigStateFilter logsearchUserConfigFilter() { + return new LogsearchUserConfigStateFilter(userConfigRequestMatcher(), solrUserConfigState, solrUserPropsConfig); + } + + @Bean public RequestMatcher requestMatcher() { List<RequestMatcher> matchers = Lists.newArrayList(); matchers.add(new AntPathRequestMatcher("/login.html")); @@ -141,4 +184,16 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { return new OrRequestMatcher(matchers); } + public RequestMatcher serviceLogsRequestMatcher() { + return new AntPathRequestMatcher("/api/v1/service/logs/**"); + } + + public RequestMatcher auditLogsRequestMatcher() { + return new AntPathRequestMatcher("/api/v1/audit/logs/**"); + } + + public RequestMatcher userConfigRequestMatcher() { + return new AntPathRequestMatcher("/api/v1/userconfig/**"); + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java index d270b13..ace278f 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java @@ -18,6 +18,7 @@ */ package org.apache.ambari.logsearch.conf; +import org.apache.zookeeper.data.ACL; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; @@ -54,6 +55,12 @@ public class SolrAuditLogPropsConfig implements SolrPropsConfig { @Value("${logsearch.collection.audit.logs.replication.factor:1}") private Integer replicationFactor; + @Value("#{ACLPropertiesSplitter.parseAcls('${logsearch.solr.audit.logs.zk.acls:}')}") + private List<ACL> zkAcls; + + @Value("${logsearch.solr.audit.logs.config_set.folder:/etc/ambari-logsearch-portal/conf/solr_configsets}") + private String configSetFolder; + @Override public String getSolrUrl() { return solrUrl; @@ -124,6 +131,26 @@ public class SolrAuditLogPropsConfig implements SolrPropsConfig { this.splitInterval = splitInterval; } + @Override + public List<ACL> getZkAcls() { + return zkAcls; + } + + @Override + public void setZkAcls(List<ACL> zkAcls) { + this.zkAcls = zkAcls; + } + + @Override + public String getConfigSetFolder() { + return configSetFolder; + } + + @Override + public void setConfigSetFolder(String configSetFolder) { + this.configSetFolder = configSetFolder; + } + public String getRangerCollection() { return rangerCollection; } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java index fb25a63..f00e8c5 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java @@ -18,13 +18,11 @@ */ package org.apache.ambari.logsearch.conf; +import org.apache.ambari.logsearch.conf.global.SolrAuditLogsState; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.conf.global.SolrServiceLogsState; +import org.apache.ambari.logsearch.conf.global.SolrUserConfigState; import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; @@ -32,52 +30,26 @@ import org.springframework.data.solr.core.SolrTemplate; import org.springframework.data.solr.repository.config.EnableSolrRepositories; import org.springframework.scheduling.annotation.EnableScheduling; -import javax.inject.Inject; - @Configuration @EnableSolrRepositories @EnableScheduling public class SolrConfig { - private static final Logger LOG = LoggerFactory.getLogger(SolrConfig.class); - - @Inject - private SolrServiceLogPropsConfig solrServiceLogPropsConfig; - - @Inject - private SolrAuditLogPropsConfig solrAuditLogPropsConfig; - - @Inject - private SolrUserPropsConfig solrUserConfigPropsConfig; - - @Inject - private SolrKerberosConfig solrKerberosConfig; - @Bean(name = "serviceSolrTemplate") public SolrTemplate serviceSolrTemplate() { - setupSecurity(); - return new SolrTemplate(createClient( - solrServiceLogPropsConfig.getSolrUrl(), - solrServiceLogPropsConfig.getZkConnectString(), - solrServiceLogPropsConfig.getCollection())); + return null; } @Bean(name = "auditSolrTemplate") @DependsOn("serviceSolrTemplate") public SolrTemplate auditSolrTemplate() { - return new SolrTemplate(createClient( - solrAuditLogPropsConfig.getSolrUrl(), - solrAuditLogPropsConfig.getZkConnectString(), - solrAuditLogPropsConfig.getCollection())); + return null; } @Bean(name = "userConfigSolrTemplate") @DependsOn("serviceSolrTemplate") public SolrTemplate userConfigSolrTemplate() { - return new SolrTemplate(createClient( - solrUserConfigPropsConfig.getSolrUrl(), - solrUserConfigPropsConfig.getZkConnectString(), - solrUserConfigPropsConfig.getCollection())); + return null; } @Bean @@ -86,26 +58,19 @@ public class SolrConfig { return new SolrSchemaFieldDao(); } - private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) { - if (StringUtils.isNotEmpty(zookeeperConnectString)) { - CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString); - cloudSolrClient.setDefaultCollection(defaultCollection); - return cloudSolrClient; - } else if (StringUtils.isNotEmpty(solrUrl)) { - throw new UnsupportedOperationException("Currently only cloud mode is supported. Set zookeeper connect string."); - } - throw new IllegalStateException( - "Solr url or zookeeper connection string is missing. collection: " + defaultCollection); + @Bean(name = "solrServiceLogsState") + public SolrCollectionState solrServiceLogsState() { + return new SolrServiceLogsState(); + } + + @Bean(name = "solrAuditLogsState") + public SolrCollectionState solrAuditLogsState() { + return new SolrAuditLogsState(); } - private void setupSecurity() { - String jaasFile = solrKerberosConfig.getJaasFile(); - boolean securityEnabled = solrKerberosConfig.isEnabled(); - if (securityEnabled) { - System.setProperty("java.security.auth.login.config", jaasFile); - HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); - LOG.info("setupSecurity() called for kerberos configuration, jaas file: " + jaasFile); - } + @Bean(name = "solrUserConfigState") + public SolrCollectionState solrUserConfigState() { + return new SolrUserConfigState(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java index 82dd7df..7d37efd 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java @@ -18,8 +18,11 @@ */ package org.apache.ambari.logsearch.conf; +import org.apache.zookeeper.data.ACL; import org.springframework.beans.factory.annotation.Value; +import java.util.List; + public abstract class SolrConnectionPropsConfig implements SolrPropsConfig { @Value("${logsearch.solr.url:}") private String solrUrl; @@ -27,6 +30,12 @@ public abstract class SolrConnectionPropsConfig implements SolrPropsConfig { @Value("${logsearch.solr.zk_connect_string:}") private String zkConnectString; + @Value("#{ACLPropertiesSplitter.parseAcls('${logsearch.solr.zk.acls:}')}") + private List<ACL> zkAcls; + + @Value("${logsearch.solr.config_set.folder:/etc/ambari-logsearch-portal/conf/solr_configsets}") + private String configSetFolder; + @Override public String getSolrUrl() { return solrUrl; @@ -46,4 +55,24 @@ public abstract class SolrConnectionPropsConfig implements SolrPropsConfig { public void setZkConnectString(String zkConnectString) { this.zkConnectString = zkConnectString; } + + @Override + public List<ACL> getZkAcls() { + return zkAcls; + } + + @Override + public void setZkAcls(List<ACL> zkAcls) { + this.zkAcls = zkAcls; + } + + @Override + public String getConfigSetFolder() { + return configSetFolder; + } + + @Override + public void setConfigSetFolder(String configSetFolder) { + this.configSetFolder = configSetFolder; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java index f80d6e5..ceddf7e 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java @@ -18,6 +18,10 @@ */ package org.apache.ambari.logsearch.conf; +import org.apache.zookeeper.data.ACL; + +import java.util.List; + public interface SolrPropsConfig { String getSolrUrl(); @@ -46,4 +50,12 @@ public interface SolrPropsConfig { String getSplitInterval(); void setSplitInterval(String splitInterval); + + List<ACL> getZkAcls(); + + void setZkAcls(List<ACL> zkAcls); + + String getConfigSetFolder(); + + void setConfigSetFolder(String configSetFolder); } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java new file mode 100644 index 0000000..546a5dc --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf.global; + +import javax.inject.Named; + +@Named +public class SolrAuditLogsState implements SolrCollectionState { + + private volatile boolean znodeReady; + private volatile boolean solrCollectionReady; + private volatile boolean solrAliasReady; + private volatile boolean configurationUploaded; + + @Override + public boolean isZnodeReady() { + return znodeReady; + } + + @Override + public void setZnodeReady(boolean znodeAvailable) { + this.znodeReady = znodeAvailable; + } + + @Override + public boolean isSolrCollectionReady() { + return solrCollectionReady; + } + + @Override + public void setSolrCollectionReady(boolean solrCollectionReady) { + this.solrCollectionReady = solrCollectionReady; + } + + @Override + public boolean isConfigurationUploaded() { + return configurationUploaded; + } + + @Override + public void setConfigurationUploaded(boolean configurationUploaded) { + this.configurationUploaded = configurationUploaded; + } + + public boolean isSolrAliasReady() { + return solrAliasReady; + } + + public void setSolrAliasReady(boolean solrAliasReady) { + this.solrAliasReady = solrAliasReady; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java new file mode 100644 index 0000000..5885611 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf.global; + +public interface SolrCollectionState { + boolean isZnodeReady(); + + void setZnodeReady(boolean znodeAvailable); + + boolean isSolrCollectionReady(); + + void setSolrCollectionReady(boolean solrCollectionCreated); + + boolean isConfigurationUploaded(); + + void setConfigurationUploaded(boolean configurationUploaded); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java new file mode 100644 index 0000000..60eafc5 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf.global; + +import javax.inject.Named; + +@Named +public class SolrServiceLogsState implements SolrCollectionState { + + private volatile boolean znodeReady; + private volatile boolean solrCollectionReady; + private volatile boolean configurationUploaded; + + @Override + public boolean isZnodeReady() { + return znodeReady; + } + + @Override + public void setZnodeReady(boolean znodeAvailable) { + this.znodeReady = znodeAvailable; + } + + @Override + public boolean isSolrCollectionReady() { + return solrCollectionReady; + } + + @Override + public void setSolrCollectionReady(boolean solrCollectionReady) { + this.solrCollectionReady = solrCollectionReady; + } + + @Override + public boolean isConfigurationUploaded() { + return configurationUploaded; + } + + @Override + public void setConfigurationUploaded(boolean configurationUploaded) { + this.configurationUploaded = configurationUploaded; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java new file mode 100644 index 0000000..06af14c --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf.global; + +import javax.inject.Named; + +@Named +public class SolrUserConfigState implements SolrCollectionState { + + private volatile boolean znodeReady; + private volatile boolean solrCollectionReady; + private volatile boolean configurationUploaded; + + @Override + public boolean isZnodeReady() { + return znodeReady; + } + + @Override + public void setZnodeReady(boolean znodeAvailable) { + this.znodeReady = znodeAvailable; + } + + @Override + public boolean isSolrCollectionReady() { + return solrCollectionReady; + } + + @Override + public void setSolrCollectionReady(boolean solrCollectionReady) { + this.solrCollectionReady = solrCollectionReady; + } + + @Override + public boolean isConfigurationUploaded() { + return configurationUploaded; + } + + @Override + public void setConfigurationUploaded(boolean configurationUploaded) { + this.configurationUploaded = configurationUploaded; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java new file mode 100644 index 0000000..34e1bec --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.configurer; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.dao.UserConfigSolrDao; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LogfeederFilterConfigurer implements SolrConfigurer { + + private static final Logger LOG = LoggerFactory.getLogger(LogfeederFilterConfigurer.class); + + private static final int SETUP_RETRY_SECOND = 10; + + private final UserConfigSolrDao userConfigSolrDao; + + public LogfeederFilterConfigurer(final UserConfigSolrDao userConfigSolrDao) { + this.userConfigSolrDao = userConfigSolrDao; + } + + @Override + public void start() { + final SolrPropsConfig solrPropsConfig = userConfigSolrDao.getSolrPropsConfig(); + final SolrCollectionState state = userConfigSolrDao.getSolrCollectionState(); + Thread setupFiltersThread = new Thread("logfeeder_filter_setup") { + @Override + public void run() { + LOG.info("logfeeder_filter_setup thread started (to upload logfeeder config)"); + while (true) { + int retryCount = 0; + try { + retryCount++; + Thread.sleep(SETUP_RETRY_SECOND * 1000); + if (state.isSolrCollectionReady()) { + LOG.info("Tries to initialize logfeeder filters in '{}' collection", solrPropsConfig.getCollection()); + userConfigSolrDao.getUserFilter(); + break; + } + } catch (Exception e) { + LOG.error("Not able to save logfeeder filter while initialization, retryCount=" + retryCount, e); + } + } + } + }; + setupFiltersThread.setDaemon(true); + setupFiltersThread.start(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java new file mode 100644 index 0000000..c80a10d --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.configurer; + +import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrAuditLogsState; +import org.apache.ambari.logsearch.dao.AuditSolrDao; +import org.apache.ambari.logsearch.handler.ListCollectionHandler; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +public class SolrAuditAliasConfigurer implements SolrConfigurer { + + private static final Logger LOG = LoggerFactory.getLogger(SolrAuditAliasConfigurer.class); + + private static final int ALIAS_SETUP_RETRY_SECOND = 30 * 60; + + private final AuditSolrDao auditSolrDao; + + public SolrAuditAliasConfigurer(final AuditSolrDao auditSolrDao) { + this.auditSolrDao = auditSolrDao; + } + + @Override + public void start() { + final SolrAuditLogPropsConfig solrPropsConfig = (SolrAuditLogPropsConfig) auditSolrDao.getSolrPropsConfig(); + final SolrAuditLogsState state = (SolrAuditLogsState) auditSolrDao.getSolrCollectionState(); + final Collection<String> collectionListIn = + Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim()); + + if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0) { + LOG.info("Will not create alias {} for {}", solrPropsConfig.getAliasNameIn(), collectionListIn.toString()); + return; + } + + LOG.info("setupAlias " + solrPropsConfig.getAliasNameIn() + " for " + collectionListIn.toString()); + // Start a background thread to do setup + Thread setupThread = new Thread("setup_alias_" + solrPropsConfig.getAliasNameIn()) { + @Override + public void run() { + LOG.info("Started monitoring thread to check availability of Solr server. alias=" + solrPropsConfig.getAliasNameIn() + + ", collections=" + collectionListIn.toString()); + int retryCount = 0; + while (true) { + if (state.isSolrCollectionReady()) { + try { + CloudSolrClient solrClient = auditSolrDao.getSolrClient(); + int count = createAlias(solrClient, solrPropsConfig.getAliasNameIn(), collectionListIn); + if (count > 0) { + solrClient.setDefaultCollection(solrPropsConfig.getAliasNameIn()); + if (count == collectionListIn.size()) { + LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " + + "Collections=" + collectionListIn); + state.setSolrAliasReady(true); + break; + } + } else { + LOG.warn("Not able to create alias=" + solrPropsConfig.getAliasNameIn() + ", retryCount=" + retryCount); + } + } catch (Exception e) { + LOG.error("Error setting up alias=" + solrPropsConfig.getAliasNameIn(), e); + } + } + try { + Thread.sleep(ALIAS_SETUP_RETRY_SECOND * 1000); + } catch (InterruptedException sleepInterrupted) { + LOG.info("Sleep interrupted while setting up alias " + solrPropsConfig.getAliasNameIn()); + break; + } + retryCount++; + } + } + }; + setupThread.setDaemon(true); + setupThread.start(); + } + + private int createAlias(final CloudSolrClient solrClient, String aliasNameIn, Collection<String> collectionListIn) + throws SolrServerException, IOException { + List<String> collectionToAdd = new ArrayList<>(); + try { + collectionToAdd = new ListCollectionHandler().handle(solrClient, null); + } catch (Exception e) { + LOG.error("Invalid state during getting collections for creating alias"); + } + collectionToAdd.retainAll(collectionListIn); + + String collectionsCSV = null; + if (!collectionToAdd.isEmpty()) { + collectionsCSV = StringUtils.join(collectionToAdd, ','); + CollectionAdminRequest.CreateAlias aliasCreateRequest = new CollectionAdminRequest.CreateAlias(); + aliasCreateRequest.setAliasName(aliasNameIn); + aliasCreateRequest.setAliasedCollections(collectionsCSV); + CollectionAdminResponse createResponse = aliasCreateRequest.process(solrClient); + if (createResponse.getStatus() != 0) { + LOG.error("Error creating alias. alias=" + aliasNameIn + ", collectionList=" + collectionsCSV + + ", response=" + createResponse); + return 0; + } + } + if (collectionToAdd.size() == collectionListIn.size()) { + LOG.info("Created alias for all collections. alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV); + } else { + LOG.info("Created alias for " + collectionToAdd.size() + " out of " + collectionListIn.size() + " collections. " + + "alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV); + } + return collectionToAdd.size(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java new file mode 100644 index 0000000..7edc6aa --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.configurer; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.dao.SolrDaoBase; +import org.apache.ambari.logsearch.handler.ACLHandler; +import org.apache.ambari.logsearch.handler.CreateCollectionHandler; +import org.apache.ambari.logsearch.handler.ListCollectionHandler; +import org.apache.ambari.logsearch.handler.ReloadCollectionHandler; +import org.apache.ambari.logsearch.handler.UploadConfigurationHandler; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.solr.core.SolrTemplate; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class SolrCollectionConfigurer implements SolrConfigurer { + + private Logger LOG = LoggerFactory.getLogger(SolrCollectionConfigurer.class); + + private static final int SETUP_RETRY_SECOND = 10; + private static final int SESSION_TIMEOUT = 15000; + private static final int CONNECTION_TIMEOUT = 30000; + + private final SolrDaoBase solrDaoBase; + + public SolrCollectionConfigurer(final SolrDaoBase solrDaoBase) { + this.solrDaoBase = solrDaoBase; + } + + @Override + public void start() { + setupSecurity(); + final SolrPropsConfig solrPropsConfig = solrDaoBase.getSolrPropsConfig(); + final SolrCollectionState state = solrDaoBase.getSolrCollectionState(); + final String separator = FileSystems.getDefault().getSeparator(); + final String localConfigSetLocation = String.format("%s%s%s%sconf", solrPropsConfig.getConfigSetFolder(), separator, + solrPropsConfig.getConfigName(), separator); + final File configSetFolder = new File(localConfigSetLocation); + if (!configSetFolder.exists()) { // show exception only once during startup + throw new RuntimeException(String.format("Cannot load config set location: %s", localConfigSetLocation)); + } + Thread setupThread = new Thread("setup_collection_" + solrPropsConfig.getCollection()) { + @Override + public void run() { + LOG.info("Started monitoring thread to check availability of Solr server. collection=" + solrPropsConfig.getCollection()); + while (!stopSetupCondition(state)) { + int retryCount = 0; + try { + retryCount++; + Thread.sleep(SETUP_RETRY_SECOND * 1000); + openZkConnectionAndUpdateStatus(state, solrPropsConfig); + if (solrDaoBase.getSolrTemplate() == null) { + solrDaoBase.setSolrTemplate(createSolrTemplate(solrPropsConfig)); + } + CloudSolrClient cloudSolrClient = (CloudSolrClient) solrDaoBase.getSolrTemplate().getSolrClient(); + boolean reloadCollectionNeeded = uploadConfigurationsIfNeeded(cloudSolrClient, configSetFolder, state, solrPropsConfig); + checkSolrStatus(cloudSolrClient); + createCollectionsIfNeeded(cloudSolrClient, state, solrPropsConfig, reloadCollectionNeeded); + } catch (Exception e) { + retryCount++; + LOG.error("Error setting collection. collection=" + solrPropsConfig.getCollection() + ", retryCount=" + retryCount, e); + } + } + } + }; + setupThread.setDaemon(true); + setupThread.start(); + } + + private boolean uploadConfigurationsIfNeeded(CloudSolrClient cloudSolrClient, File configSetFolder, SolrCollectionState state, SolrPropsConfig solrPropsConfig) throws Exception { + boolean reloadCollectionNeeded = new UploadConfigurationHandler(configSetFolder).handle(cloudSolrClient, solrPropsConfig); + if (!state.isConfigurationUploaded()) { + state.setConfigurationUploaded(true); + } + return reloadCollectionNeeded; + } + + public boolean stopSetupCondition(SolrCollectionState state) { + return state.isSolrCollectionReady(); + } + + public SolrTemplate createSolrTemplate(SolrPropsConfig solrPropsConfig) { + return new SolrTemplate(createClient( + solrPropsConfig.getSolrUrl(), + solrPropsConfig.getZkConnectString(), + solrPropsConfig.getCollection())); + } + + private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) { + if (StringUtils.isNotEmpty(zookeeperConnectString)) { + CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString); + cloudSolrClient.setDefaultCollection(defaultCollection); + return cloudSolrClient; + } else if (StringUtils.isNotEmpty(solrUrl)) { + throw new UnsupportedOperationException("Currently only cloud mode is supported. Set zookeeper connect string."); + } + throw new IllegalStateException( + "Solr url or zookeeper connection string is missing. collection: " + defaultCollection); + } + + private void setupSecurity() { + String jaasFile = solrDaoBase.getSolrKerberosConfig().getJaasFile(); + boolean securityEnabled = solrDaoBase.getSolrKerberosConfig().isEnabled(); + if (securityEnabled) { + System.setProperty("java.security.auth.login.config", jaasFile); + HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); + LOG.info("setupSecurity() called for kerberos configuration, jaas file: " + jaasFile); + } + } + + private void openZkConnectionAndUpdateStatus(final SolrCollectionState state, final SolrPropsConfig solrPropsConfig) throws Exception { + ZooKeeper zkClient = null; + try { + LOG.info("Checking that Znode ('{}') is ready or not... ", solrPropsConfig.getZkConnectString()); + zkClient = openZookeeperConnection(solrPropsConfig); + if (!state.isZnodeReady()) { + LOG.info("State change: Zookeeper ZNode is available for {}", solrPropsConfig.getZkConnectString()); + state.setZnodeReady(true); + } + } catch (Exception e) { + LOG.error("Error occurred during the creation of zk client (connection string: {})", solrPropsConfig.getZkConnectString()); + throw e; + } finally { + try { + if (zkClient != null) { + zkClient.close(); + } + } catch (Exception e) { + LOG.error("Could not close zk connection properly.", e); + } + } + } + + private ZooKeeper openZookeeperConnection(final SolrPropsConfig solrPropsConfig) throws InterruptedException, IOException { + final CountDownLatch connSignal = new CountDownLatch(1); + ZooKeeper zooKeeper = new ZooKeeper(solrPropsConfig.getZkConnectString(), SESSION_TIMEOUT, new Watcher() { + public void process(WatchedEvent event) { + if (event.getState() == Event.KeeperState.SyncConnected) { + connSignal.countDown(); + } + } + }); + connSignal.await(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); + return zooKeeper; + } + + private boolean checkSolrStatus(CloudSolrClient cloudSolrClient) { + int waitDurationMS = 3 * 60 * 1000; + boolean status = false; + try { + long beginTimeMS = System.currentTimeMillis(); + long waitIntervalMS = 2000; + int pingCount = 0; + while (true) { + pingCount++; + try { + List<String> collectionList = new ListCollectionHandler().handle(cloudSolrClient, null); + if (collectionList != null) { + LOG.info("checkSolrStatus(): Solr getCollections() is success. collectionList=" + collectionList); + status = true; + break; + } + } catch (Exception ex) { + LOG.error("Error while doing Solr check", ex); + } + if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) { + LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " + + "If you are using alias, then you might have to restart LogSearch after Solr is up and running."); + break; + } else { + LOG.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " + + "Will sleep for " + waitIntervalMS + " ms and try again."); + } + Thread.sleep(waitIntervalMS); + + } + } catch (Throwable t) { + LOG.error("Seems Solr is not up."); + } + return status; + } + + private void createCollectionsIfNeeded(CloudSolrClient solrClient, SolrCollectionState state, SolrPropsConfig solrPropsConfig, boolean reloadCollectionNeeded) { + try { + List<String> allCollectionList = new ListCollectionHandler().handle(solrClient, null); + boolean collectionCreated = new CreateCollectionHandler(allCollectionList).handle(solrClient, solrPropsConfig); + boolean collectionReloaded = true; + if (reloadCollectionNeeded) { + collectionReloaded = new ReloadCollectionHandler().handle(solrClient, solrPropsConfig); + } + boolean aclsUpdated = new ACLHandler().handle(solrClient, solrPropsConfig); + if (!state.isSolrCollectionReady() && collectionCreated && collectionReloaded && aclsUpdated) { + state.setSolrCollectionReady(true); + } + } catch (Exception ex) { + LOG.error("Error during creating/updating collection. collectionName=" + solrPropsConfig.getCollection(), ex); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java new file mode 100644 index 0000000..67cb9d1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.configurer; + +interface SolrConfigurer { + void start(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java index 309687d..d058383 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java @@ -25,6 +25,10 @@ import javax.inject.Named; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.configurer.SolrAuditAliasConfigurer; +import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.data.solr.core.SolrTemplate; @@ -42,13 +46,8 @@ public class AuditSolrDao extends SolrDaoBase { private SolrTemplate auditSolrTemplate; @Inject - private SolrAliasDao solrAliasDao; - - @Inject - private SolrCollectionDao solrCollectionDao; - - @Inject - private SolrSchemaFieldDao solrSchemaFieldDao; + @Named("solrAuditLogsState") + private SolrCollectionState solrAuditLogsState; public AuditSolrDao() { super(LogType.AUDIT); @@ -59,26 +58,37 @@ public class AuditSolrDao extends SolrDaoBase { return auditSolrTemplate; } + @Override + public void setSolrTemplate(SolrTemplate solrTemplate) { + this.auditSolrTemplate = solrTemplate; + } + @PostConstruct public void postConstructor() { String aliasNameIn = solrAuditLogPropsConfig.getAliasNameIn(); String rangerAuditCollection = solrAuditLogPropsConfig.getRangerCollection(); try { - solrCollectionDao.checkSolrStatus(getSolrClient()); + new SolrCollectionConfigurer(this).start(); boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection)); - solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig); if (createAlias) { - solrAliasDao.setupAlias(getSolrClient(), solrAuditLogPropsConfig); + new SolrAuditAliasConfigurer(this).start(); } - solrSchemaFieldDao.auditCollectionSetUp(); } catch (Exception e) { LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" + - solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e); + solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e); } } - public SolrSchemaFieldDao getSolrSchemaFieldDao() { - return solrSchemaFieldDao; + @Override + public SolrCollectionState getSolrCollectionState() { + return this.solrAuditLogsState; + } + + @Override + public SolrPropsConfig getSolrPropsConfig() { + return this.solrAuditLogPropsConfig; } + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java index cf61163..37375dc 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java @@ -24,7 +24,10 @@ import javax.inject.Inject; import javax.inject.Named; import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer; import org.apache.log4j.Logger; import org.springframework.data.solr.core.SolrTemplate; @@ -34,17 +37,15 @@ public class ServiceLogsSolrDao extends SolrDaoBase { private static final Logger LOG = Logger.getLogger(ServiceLogsSolrDao.class); @Inject - private SolrCollectionDao solrCollectionDao; - - @Inject private SolrServiceLogPropsConfig solrServiceLogPropsConfig; @Inject @Named("serviceSolrTemplate") - private SolrTemplate serviceSolrTemplate; + private volatile SolrTemplate serviceSolrTemplate; @Inject - private SolrSchemaFieldDao solrSchemaFieldDao; + @Named("solrServiceLogsState") + private SolrCollectionState solrServiceLogsState; public ServiceLogsSolrDao() { super(LogType.SERVICE); @@ -55,13 +56,16 @@ public class ServiceLogsSolrDao extends SolrDaoBase { return serviceSolrTemplate; } + @Override + public void setSolrTemplate(SolrTemplate solrTemplate) { + this.serviceSolrTemplate = solrTemplate; + } + @PostConstruct public void postConstructor() { LOG.info("postConstructor() called."); try { - solrCollectionDao.checkSolrStatus(getSolrClient()); - solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig); - solrSchemaFieldDao.serviceCollectionSetUp(); + new SolrCollectionConfigurer(this).start(); } catch (Exception e) { LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl() + ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString() @@ -69,7 +73,13 @@ public class ServiceLogsSolrDao extends SolrDaoBase { } } - public SolrSchemaFieldDao getSolrSchemaFieldDao() { - return solrSchemaFieldDao; + @Override + public SolrCollectionState getSolrCollectionState() { + return solrServiceLogsState; + } + + @Override + public SolrPropsConfig getSolrPropsConfig() { + return solrServiceLogPropsConfig; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java deleted file mode 100644 index d78ff0da..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.dao; - -import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import javax.inject.Named; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -@Named -class SolrAliasDao { - - private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class); - - private static final int ALIAS_SETUP_RETRY_SECOND = 30*60; - - @Inject - private SolrCollectionDao solrCollectionDao; - - void setupAlias(final CloudSolrClient solrClient, final SolrAuditLogPropsConfig solrPropsConfig) throws Exception { - final Collection<String> collectionListIn = - Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim()); - - if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) { - LOG.info("Will not create alias " + solrPropsConfig.getAliasNameIn() + " for " + - collectionListIn.toString() + ", solrCloudClient=" + solrClient); - return; - } - - LOG.info("setupAlias " + solrPropsConfig.getAliasNameIn() + " for " + collectionListIn.toString()); - // Start a background thread to do setup - Thread setupThread = new Thread("setup_alias_" + solrPropsConfig.getAliasNameIn()) { - @Override - public void run() { - LOG.info("Started monitoring thread to check availability of Solr server. alias=" + solrPropsConfig.getAliasNameIn() + - ", collections=" + collectionListIn.toString()); - int retryCount = 0; - while (true) { - try { - int count = createAlias(solrClient, solrPropsConfig.getAliasNameIn(), collectionListIn); - if (count > 0) { - solrClient.setDefaultCollection(solrPropsConfig.getAliasNameIn()); - if (count == collectionListIn.size()) { - LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " + - "Collections=" + collectionListIn); - break; - } - } else { - LOG.warn("Not able to create alias=" + solrPropsConfig.getAliasNameIn() + ", retryCount=" + retryCount); - } - } catch (Exception e) { - LOG.error("Error setting up alias=" + solrPropsConfig.getAliasNameIn(), e); - } - try { - Thread.sleep(ALIAS_SETUP_RETRY_SECOND * 1000); - } catch (InterruptedException sleepInterrupted) { - LOG.info("Sleep interrupted while setting up alias " + solrPropsConfig.getAliasNameIn()); - break; - } - retryCount++; - } - } - }; - setupThread.setDaemon(true); - setupThread.start(); - } - - private int createAlias(final CloudSolrClient solrClient, String aliasNameIn, Collection<String> collectionListIn) - throws SolrServerException, IOException { - List<String> collectionToAdd = solrCollectionDao.getCollections(solrClient); - collectionToAdd.retainAll(collectionListIn); - - String collectionsCSV = null; - if (!collectionToAdd.isEmpty()) { - collectionsCSV = StringUtils.join(collectionToAdd, ','); - CollectionAdminRequest.CreateAlias aliasCreateRequest = new CollectionAdminRequest.CreateAlias(); - aliasCreateRequest.setAliasName(aliasNameIn); - aliasCreateRequest.setAliasedCollections(collectionsCSV); - CollectionAdminResponse createResponse = aliasCreateRequest.process(solrClient); - if (createResponse.getStatus() != 0) { - LOG.error("Error creating alias. alias=" + aliasNameIn + ", collectionList=" + collectionsCSV - + ", response=" + createResponse); - return 0; - } - } - if ( collectionToAdd.size() == collectionListIn.size()) { - LOG.info("Created alias for all collections. alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV); - } else { - LOG.info("Created alias for " + collectionToAdd.size() + " out of " + collectionListIn.size() + " collections. " + - "alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV); - } - return collectionToAdd.size(); - } -}
