RANGER-728: Update Solr script to resolve issues with ZK and creating collection (cherry-picked commit 269617d5dbf13fcbf9600efca72bc5a803f49a92)
Signed-off-by: Velmurugan Periasamy <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/262da5a4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/262da5a4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/262da5a4 Branch: refs/heads/ranger-0.5 Commit: 262da5a45405927a0b2ace9d0bab283aaa947529 Parents: f48ad91 Author: Don Bosco Durai <[email protected]> Authored: Wed Dec 9 16:23:11 2015 -0800 Committer: Velmurugan Periasamy <[email protected]> Committed: Fri Dec 11 00:18:49 2015 -0500 ---------------------------------------------------------------------- .../create_hdfs_folders_for_audit_secure.sh | 4 +- .../solr_for_audit_setup/conf/managed-schema | 92 + .../solr_for_audit_setup/conf/schema.xml | 118 -- .../solr_for_audit_setup/conf/solrconfig.xml | 13 + .../solr_for_audit_setup/conf/solrconfig.xml.j2 | 1878 ++++++++++++++++++ .../solr_for_audit_setup/install.properties | 3 + .../resources/log4j.properties.j2 | 40 + .../resources/log4j.properties.template | 39 - .../contrib/solr_for_audit_setup/setup.sh | 31 +- .../scripts/add_ranger_audits_conf_to_zk.sh.j2 | 63 + .../add_ranger_audits_conf_to_zk.sh.template | 63 - .../create_ranger_audits_collection.sh.j2 | 33 + .../create_ranger_audits_collection.sh.template | 33 - .../solr_cloud/scripts/solr.in.sh.j2 | 116 ++ .../solr_cloud/scripts/solr.sh.j2 | 21 + .../solr_cloud/scripts/start_solr.sh.j2 | 32 + .../solr_cloud/scripts/start_solr.sh.template | 39 - .../solr_cloud/scripts/stop_solr.sh.j2 | 33 + .../solr_cloud/scripts/stop_solr.sh.template | 35 - .../solr_for_audit_setup/solr_cloud/solr.xml.j2 | 26 + .../solr_cloud/solr.xml.template | 26 - .../ranger_audits/core.properties.j2 | 20 + .../ranger_audits/core.properties.template | 20 - .../solr_standalone/scripts/solr.in.sh.j2 | 116 ++ .../solr_standalone/scripts/solr.sh.j2 | 21 + .../solr_standalone/scripts/start_solr.sh.j2 | 33 + .../scripts/start_solr.sh.template | 38 - .../solr_standalone/scripts/stop_solr.sh.j2 | 33 + .../scripts/stop_solr.sh.template | 35 - 29 files changed, 2593 insertions(+), 461 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/262da5a4/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh ---------------------------------------------------------------------- diff --git a/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh b/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh index 12a4c93..9a5bdd5 100755 --- a/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh +++ b/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Usage: Run this script as user hdfs or the HDFS admin user. +#Usage: Use this script in kerberos enabled hadoop only. Run this script after kinit'ing as hdfs user #This script creates the folders in HDFS required by Apache Ranger for writing Audit records -#Note 1: Use this script only for non-kerberos environment. In non-kerberos environment, Ranger KMS writes the audit logs as user "HTTP" +#Note 1: Use this script only for kerberos environment. In kerberos environment, Ranger KMS writes the audit logs as user "HTTP" #Note 2: Please update the below variables according to your environment HBASE_USER_GROUP=hbase:hbase http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/262da5a4/security-admin/contrib/solr_for_audit_setup/conf/managed-schema ---------------------------------------------------------------------- diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema new file mode 100644 index 0000000..7b9769a --- /dev/null +++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<schema name="ranger-audit-schema" version="1.5"> + <uniqueKey>id</uniqueKey> + <fieldType name="binary" class="solr.BinaryField"/> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> + <fieldType name="booleans" class="solr.BoolField" multiValued="true" sortMissingLast="true"/> + <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="ignored" class="solr.StrField" multiValued="true" indexed="false" stored="false"/> + <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="key_lower_case" class="solr.TextField" sortMissingLast="true" omitNorms="true"> + <analyzer> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="random" class="solr.RandomSortField" indexed="true"/> + <fieldType name="string" class="solr.StrField" sortMissingLast="true"/> + <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/> + <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" multiValued="true" positionIncrementGap="0"/> + <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/> + <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> + <fieldType name="text_std_token_lower_case" class="solr.TextField" multiValued="true" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + </analyzer> + </fieldType> + <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/> + <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> + <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/> + <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> + <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/> + <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" multiValued="true" positionIncrementGap="0"/> + <field name="_expire_at_" type="tdate" multiValued="false" stored="true"/> + <field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/> + <field name="_version_" type="long" indexed="true" stored="true"/> + <field name="access" type="key_lower_case" multiValued="false"/> + <field name="action" type="key_lower_case" multiValued="false"/> + <field name="agent" type="key_lower_case" multiValued="false"/> + <field name="agentHost" type="key_lower_case" multiValued="false"/> + <field name="cliIP" type="key_lower_case" multiValued="false"/> + <field name="cliType" type="key_lower_case" multiValued="false"/> + <field name="cluster" type="key_lower_case" multiValued="false"/> + <field name="reqContext" type="key_lower_case" multiValued="true"/> + <field name="enforcer" type="key_lower_case" multiValued="false"/> + <field name="event_count" type="tlong" multiValued="false" docValues="true"/> + <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/> + <field name="evtTime" type="tdate"/> + <field name="id" type="string" multiValued="false" indexed="true" required="true" stored="true"/> + <field name="logType" type="key_lower_case" multiValued="false"/> + <field name="policy" type="tlong" docValues="true"/> + <field name="proxyUsers" type="key_lower_case" multiValued="true"/> + <field name="reason" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/> + <field name="repo" type="key_lower_case" multiValued="false"/> + <field name="repoType" type="tint" multiValued="false" docValues="true"/> + <field name="req_caller_id" type="key_lower_case" multiValued="false"/> + <field name="req_self_id" type="key_lower_case" multiValued="false"/> + <field name="reqData" type="text_std_token_lower_case" multiValued="false"/> + <field name="reqUser" type="key_lower_case" multiValued="false"/> + <field name="resType" type="key_lower_case" multiValued="false"/> + <field name="resource" type="key_lower_case" multiValued="false"/> + <field name="result" type="tint" multiValued="false"/> + <field name="seq_num" type="tlong" multiValued="false" docValues="true"/> + <field name="sess" type="key_lower_case" multiValued="false"/> + <field name="tags" type="key_lower_case" multiValued="true"/> + <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/> + <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/> +</schema> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/262da5a4/security-admin/contrib/solr_for_audit_setup/conf/schema.xml ---------------------------------------------------------------------- diff --git a/security-admin/contrib/solr_for_audit_setup/conf/schema.xml b/security-admin/contrib/solr_for_audit_setup/conf/schema.xml deleted file mode 100644 index df56974..0000000 --- a/security-admin/contrib/solr_for_audit_setup/conf/schema.xml +++ /dev/null @@ -1,118 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- Trimmed schema.xml to include only the fields used by RangerAudit and also to make it more manageable --> -<schema name="ranger-audit-schema" version="1.5"> - <fields> - <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> - <field name="_version_" type="long" indexed="true" stored="true"/> - - <field name="resource" type="key_lower_case" multiValued="false"/> - <field name="cluster" type="key_lower_case" multiValued="false"/> - <field name="resType" type="key_lower_case" multiValued="false"/> - <field name="reqUser" type="key_lower_case" multiValued="false"/> - <field name="evtTime" type="tdate"/> - <field name="policy" type="tlong" docValues="true"/> - <field name="sess" type="key_lower_case" multiValued="false"/> - <field name="access" type="key_lower_case" multiValued="false"/> - <field name="result" type="tint" multiValued="false"/> - <field name="reason" type="text_std_token_lower_case" omitNorms="false" multiValued="false"/> - <field name="enforcer" type="key_lower_case" multiValued="false"/> - <field name="repo" type="key_lower_case" multiValued="false"/> - <field name="cliIP" type="key_lower_case" multiValued="false"/> - <field name="cliType" type="key_lower_case" multiValued="false"/> - <field name="action" type="key_lower_case" multiValued="false"/> - <field name="agent" type="key_lower_case" multiValued="false"/> - <field name="agentHost" type="key_lower_case" multiValued="false"/> - <field name="logType" type="key_lower_case" multiValued="false"/> - <field name="repoType" type="tint" multiValued="false" docValues="true"/> - <field name="reqData" type="text_std_token_lower_case" multiValued="false"/> - <field name="seq_num" type="tlong" multiValued="false" docValues="true"/> - <field name="tags" type="key_lower_case" multiValued="true"/> - <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/> - <field name="event_count" type="tlong" multiValued="false" docValues="true"/> - <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/> - - <field name="text" type="text_std_token_lower_case" indexed="true" stored="false" multiValued="true"/> - </fields> - - <uniqueKey>id</uniqueKey> - - <types> - - <!-- Updated from text_general and having only StandardTokenizer and LowerCaseFilter--> - <fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100" multiValued="true"> - <analyzer> - <tokenizer class="solr.StandardTokenizerFactory"/> - <filter class="solr.LowerCaseFilterFactory" /> - </analyzer> - </fieldType> - - <!-- Keyword tokenizer won't split the string. So is like full string search --> - <fieldType name="key_lower_case" class="solr.TextField" - sortMissingLast="true" omitNorms="true"> - <analyzer> - <tokenizer class="solr.KeywordTokenizerFactory"/> - <filter class="solr.LowerCaseFilterFactory" /> - </analyzer> - </fieldType> - - <!-- The StrField type is not analyzed, but indexed/stored verbatim. - It supports doc values but in that case the field needs to be - single-valued and either required or have a default value. - --> - <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> - - <!-- boolean type: "true" or "false" --> - <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> - - <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/> - - <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/> - <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/> - <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/> - <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/> - - <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/> - <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/> - <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/> - <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/> - - <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0" multiValued="true"/> - <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0" multiValued="true"/> - <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0" multiValued="true"/> - <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0" multiValued="true"/> - <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/> - <!-- A Trie based date field for faster date range queries and date faceting. --> - <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/> - <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0" multiValued="true"/> - <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings --> - <fieldtype name="binary" class="solr.BinaryField"/> - <fieldType name="random" class="solr.RandomSortField" indexed="true" /> - <!-- A text field that only splits on whitespace for exact matching of words --> - <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> - <analyzer> - <tokenizer class="solr.WhitespaceTokenizerFactory"/> - </analyzer> - </fieldType> - <!-- since fields of this type are by default not stored or indexed, - any data added to them will be ignored outright. --> - <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> - - </types> -</schema> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/262da5a4/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml ---------------------------------------------------------------------- diff --git a/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml b/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml index 0991eba..7e71f9a 100644 --- a/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml +++ b/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml @@ -1642,6 +1642,19 @@ See http://wiki.apache.org/solr/GuessingFieldTypes --> <updateRequestProcessorChain name="add-unknown-fields-to-the-schema"> + <processor class="solr.DefaultValueUpdateProcessorFactory"> + <str name="fieldName">_ttl_</str> + <str name="value">+1095DAYS</str> + </processor> + <processor class="solr.processor.DocExpirationUpdateProcessorFactory"> + <int name="autoDeletePeriodSeconds">300</int> + <str name="ttlFieldName">_ttl_</str> + <str name="expirationFieldName">_expire_at_</str> + </processor> + <processor class="solr.FirstFieldValueUpdateProcessorFactory"> + <str name="fieldName">_expire_at_</str> + </processor> + <processor class="solr.RemoveBlankFieldUpdateProcessorFactory"/> <processor class="solr.ParseBooleanFieldUpdateProcessorFactory"/> <processor class="solr.ParseLongFieldUpdateProcessorFactory"/>
