Repository: incubator-ranger Updated Branches: refs/heads/master 0d9a31a93 -> 399ed34b2
RANGER-246 - Ranger Admin side support Kafka integration Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/10b57288 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/10b57288 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/10b57288 Branch: refs/heads/master Commit: 10b57288904da5e832300850e799f8debb4cc909 Parents: be34cc2 Author: Don Bosco Durai <[email protected]> Authored: Thu Apr 2 12:18:33 2015 -0700 Committer: Don Bosco Durai <[email protected]> Committed: Thu Apr 2 12:18:33 2015 -0700 ---------------------------------------------------------------------- .../service-defs/ranger-servicedef-kafka.json | 112 +++++++++ .../service-defs/ranger-servicedef-solr.json | 13 +- plugin-kafka/.gitignore | 1 + .../conf/ranger-kafka-audit-changes.cfg | 36 +++ plugin-kafka/conf/ranger-kafka-audit.xml | 235 +++++++++++++++++++ .../conf/ranger-kafka-security-changes.cfg | 26 ++ plugin-kafka/conf/ranger-kafka-security.xml | 67 ++++++ .../conf/ranger-policymgr-ssl-changes.cfg | 23 ++ plugin-kafka/conf/ranger-policymgr-ssl.xml | 63 +++++ plugin-kafka/pom.xml | 56 +++++ plugin-kafka/scripts/install.properties | 112 +++++++++ .../scripts/kafka-plugin-install.properties | 23 ++ .../kafka/authorizer/RangerKafkaAuthorizer.java | 33 +++ .../services/kafka/RangerServiceKafka.java | 78 ++++++ .../kafka/client/ServiceKafkaClient.java | 199 ++++++++++++++++ .../kafka/client/ServiceKafkaConnectionMgr.java | 55 +++++ .../services/solr/client/ServiceSolrClient.java | 2 +- pom.xml | 2 + src/main/assembly/plugin-kafka.xml | 157 +++++++++++++ 19 files changed, 1285 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json new file mode 100644 index 0000000..11b5d15 --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json @@ -0,0 +1,112 @@ +{ + "name":"kafka", + "implClass":"org.apache.ranger.services.kafka.RangerServiceKafka", + "label":"Kafka", + "description":"Apache Kafka", + "resources":[ + { + "id":1, + "name":"topic", + "type":"string", + "level":1, + "mandatory":true, + "lookupSupported":true, + "recursiveSupported":false, + "excludesSupported":true, + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions":{ + "wildCard":true, + "ignoreCase":true + }, + "validationRegEx":"", + "validationMessage":"", + "uiHint":"", + "label":"Topic", + "description":"Topic" + } + + ], + "accessTypes":[ + { + "name":"publish", + "label":"Publish" + }, + { + "name":"consume", + "label":"Consume" + }, + { + "name":"create", + "label":"Create" + }, + { + "name":"delete", + "label":"Delete" + }, + { + "name":"configure", + "label":"Configure" + }, + { + "name":"describe", + "label":"Describe" + }, + { + "name":"replicate", + "label":"Replicate" + }, + { + "name":"connect", + "label":"Connect" + } + + ], + "configs":[ + { + "name":"username", + "type":"string", + "mandatory":true, + "label":"Username" + }, + { + "name":"password", + "type":"password", + "mandatory":true, + "label":"Password" + }, + { + "name":"zookeeper.connect", + "type":"string", + "mandatory":true, + "defaultValue":"localhost:2181", + "label":"Zookeeper Connect String" + }, + { + "name":"certificate.cn", + "type":"string", + "mandatory":false, + "label":"Common Name for Certificate" + } + + ], + "enums":[ + + ], + "contextEnrichers":[ + + ], + "policyConditions":[ + { + "id": 1, + "name": "ip-range", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher", + "evaluatorOptions": { }, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "IP Address Range", + "description": "IP Address Range" + } + + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json index 19e9b63..793ec28 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json @@ -118,15 +118,14 @@ "policyConditions":[ { "id": 1, - "name":"ip-range", - "evaluator":"org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher", - "evaluatorOptions":"", + "name": "ip-range", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher", + "evaluatorOptions": { }, "validationRegEx":"", - "validationMessage":"", + "validationMessage": "", "uiHint":"", - "label":"IP Address Range", - "description":"IP Address Range" + "label": "IP Address Range", + "description": "IP Address Range" } - ] } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/.gitignore ---------------------------------------------------------------------- diff --git a/plugin-kafka/.gitignore b/plugin-kafka/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/plugin-kafka/.gitignore @@ -0,0 +1 @@ +/target http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg new file mode 100644 index 0000000..7c0c430 --- /dev/null +++ b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT_DB_JDBC_URL% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists +xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT_DB_JDBC_DRIVER% mod create-if-not-exists + +xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.directory %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.directory %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-audit.xml ---------------------------------------------------------------------- diff --git a/plugin-kafka/conf/ranger-kafka-audit.xml b/plugin-kafka/conf/ranger-kafka-audit.xml new file mode 100644 index 0000000..bff6cbf --- /dev/null +++ b/plugin-kafka/conf/ranger-kafka-audit.xml @@ -0,0 +1,235 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>xasecure.audit.is.enabled</name> + <value>true</value> + </property> + + + <!-- DB audit provider configuration --> + <property> + <name>xasecure.audit.db.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.db.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.db.batch.size</name> + <value>100</value> + </property> + + <!-- Properties whose name begin with "xasecure.audit.jpa." are used to configure JPA --> + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.url</name> + <value>jdbc:mysql://localhost:3306/ranger_audit</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.user</name> + <value>rangerlogger</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.password</name> + <value>none</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.driver</name> + <value>com.mysql.jdbc.Driver</value> + </property> + + <property> + <name>xasecure.audit.credential.provider.file</name> + <value>jceks://file/etc/ranger/kafkadev/auditcred.jceks</value> + </property> + + + + <!-- HDFS audit provider configuration --> + <property> + <name>xasecure.audit.hdfs.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.hdfs.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.queue.size</name> + <value>1048576</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.encoding</name> + <value></value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.directory</name> + <value>hdfs://NAMENODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.file</name> + <value>%hostname%-audit.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name> + <value>900</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name> + <value>86400</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.directory</name> + <value>/var/log/kafka/audit</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file</name> + <value>%time:yyyyMMdd-HHmm.ss%.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file.buffer.size.bytes</name> + <value>8192</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name> + <value>600</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.directory</name> + <value>/var/log/kafka/audit/archive</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name> + <value>10</value> + </property> + + + <!-- Log4j audit provider configuration --> + <property> + <name>xasecure.audit.log4j.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.is.async</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.solr.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.solr.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.solr.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-security-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-kafka/conf/ranger-kafka-security-changes.cfg b/plugin-kafka/conf/ranger-kafka-security-changes.cfg new file mode 100644 index 0000000..fe36616 --- /dev/null +++ b/plugin-kafka/conf/ranger-kafka-security-changes.cfg @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Change the original policy parameter to work with policy manager based. +# +# +ranger.plugin.kafka.service.name %REPOSITORY_NAME% mod create-if-not-exists + +ranger.plugin.kafka.policy.source.impl org.apache.ranger.admin.client.RangerAdminRESTClient mod create-if-not-exists + +ranger.plugin.kafka.policy.rest.url %POLICY_MGR_URL% mod create-if-not-exists +ranger.plugin.kafka.policy.rest.ssl.config.file /etc/kafka/conf/ranger-policymgr-ssl.xml mod create-if-not-exists +ranger.plugin.kafka.policy.pollIntervalMs 30000 mod create-if-not-exists +ranger.plugin.kafka.policy.cache.dir %POLICY_CACHE_FILE_PATH% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-security.xml ---------------------------------------------------------------------- diff --git a/plugin-kafka/conf/ranger-kafka-security.xml b/plugin-kafka/conf/ranger-kafka-security.xml new file mode 100644 index 0000000..b11a71a --- /dev/null +++ b/plugin-kafka/conf/ranger-kafka-security.xml @@ -0,0 +1,67 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>ranger.plugin.kafka.service.name</name> + <value>kafkadev</value> + <description> + Name of the Ranger service containing policies for this KAFKA instance + </description> + </property> + + <property> + <name>ranger.plugin.kafka.policy.source.impl</name> + <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value> + <description> + Class to retrieve policies from the source + </description> + </property> + + <property> + <name>ranger.plugin.kafka.policy.rest.url</name> + <value>http://policymanagerhost:port</value> + <description> + URL to Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.kafka.policy.rest.ssl.config.file</name> + <value>/etc/kafka/conf/ranger-policymgr-ssl.xml</value> + <description> + Path to the file containing SSL details to contact Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.kafka.policy.pollIntervalMs</name> + <value>30000</value> + <description> + How often to poll for changes in policies? + </description> + </property> + + <property> + <name>ranger.plugin.kafka.policy.cache.dir</name> + <value>/etc/ranger/kafkadev/policycache</value> + <description> + Directory where Ranger policies are cached after successful retrieval from the source + </description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg b/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg new file mode 100644 index 0000000..ec4eeab --- /dev/null +++ b/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SSL Params +# +xasecure.policymgr.clientssl.keystore %SSL_KEYSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.keystore.password %SSL_KEYSTORE_PASSWORD% mod create-if-not-exists +xasecure.policymgr.clientssl.keystore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore %SSL_TRUSTSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore.password %SSL_TRUSTSTORE_PASSWORD% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/plugin-kafka/conf/ranger-policymgr-ssl.xml b/plugin-kafka/conf/ranger-policymgr-ssl.xml new file mode 100644 index 0000000..e80f1c3 --- /dev/null +++ b/plugin-kafka/conf/ranger-policymgr-ssl.xml @@ -0,0 +1,63 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <!-- The following properties are used for 2-way SSL client server validation --> + <property> + <name>xasecure.policymgr.clientssl.keystore</name> + <value>kafkadev-clientcert.jks</value> + <description> + Java Keystore files + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.password</name> + <value>none</value> + <description> + password for keystore + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore</name> + <value>cacerts-xasecure.jks</value> + <description> + java truststore file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.password</name> + <value>none</value> + <description> + java truststore password + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.credential.file</name> + <value>jceks://file/tmp/keystore-kafkadev-ssl.jceks</value> + <description> + java keystore credential file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.credential.file</name> + <value>jceks://file/tmp/truststore-kafkadev-ssl.jceks</value> + <description> + java truststore credential file + </description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/pom.xml ---------------------------------------------------------------------- diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml new file mode 100644 index 0000000..e9ea265 --- /dev/null +++ b/plugin-kafka/pom.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>security_plugins.ranger-kafka-plugin</groupId> + <artifactId>ranger-kafka-plugin</artifactId> + <name>KAFKA Security Plugin</name> + <description>KAFKA Security Plugin</description> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <parent> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger</artifactId> + <version>0.5.0</version> + <relativePath>..</relativePath> + </parent> + <dependencies> + <dependency> + <groupId>security_plugins.ranger-plugins-common</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>security_plugins.ranger-plugins-audit</groupId> + <artifactId>ranger-plugins-audit</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>credentialbuilder</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka_2.10</artifactId> + <version>${kafka.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/scripts/install.properties ---------------------------------------------------------------------- diff --git a/plugin-kafka/scripts/install.properties b/plugin-kafka/scripts/install.properties new file mode 100644 index 0000000..506c638 --- /dev/null +++ b/plugin-kafka/scripts/install.properties @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=kafkadev +# +REPOSITORY_NAME= + +# +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=false +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME= +XAAUDIT.DB.DATABASE_NAME= +XAAUDIT.DB.USER_NAME= +XAAUDIT.DB.PASSWORD= + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit/archive +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/kafka/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/kafka/audit/archive +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit/archive + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/scripts/kafka-plugin-install.properties ---------------------------------------------------------------------- diff --git a/plugin-kafka/scripts/kafka-plugin-install.properties b/plugin-kafka/scripts/kafka-plugin-install.properties new file mode 100644 index 0000000..f4df857 --- /dev/null +++ b/plugin-kafka/scripts/kafka-plugin-install.properties @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# installation properties for this plugin + +# +# Name of the directory where the component's lib and conf directory exist. +# This location should be relative to the parent of the directory containing +# the plugin installation files. +# +COMPONENT_INSTALL_DIR_NAME=kafka http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java ---------------------------------------------------------------------- diff --git a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java new file mode 100644 index 0000000..40c2204 --- /dev/null +++ b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java @@ -0,0 +1,33 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.authorization.kafka.authorizer; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerKafkaAuthorizer /*KafkaAuthorizationPlugin*/ { + + private static final Log LOG = LogFactory.getLog(RangerKafkaAuthorizer.class); + + //private static volatile RangerKafkaPlugin kafkaPlugin = null; + + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java ---------------------------------------------------------------------- diff --git a/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java new file mode 100644 index 0000000..ea6d316 --- /dev/null +++ b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.kafka; + +import java.util.HashMap; +import java.util.List; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.services.kafka.client.ServiceKafkaClient; +import org.apache.ranger.services.kafka.client.ServiceKafkaConnectionMgr; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerServiceKafka extends RangerBaseService { + + private static final Log LOG = LogFactory.getLog(RangerServiceKafka.class); + + public RangerServiceKafka() { + super(); + } + + @Override + public void init(RangerServiceDef serviceDef, RangerService service) { + super.init(serviceDef, service); + } + + @Override + public HashMap<String, Object> validateConfig() throws Exception { + HashMap<String, Object> ret = new HashMap<String, Object>(); + String serviceName = getServiceName(); + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceKafka.validateConfig Service: (" + + serviceName + " )"); + } + if (configs != null) { + try { + ret = ServiceKafkaConnectionMgr.testConnection(serviceName, + configs); + } catch (Exception e) { + LOG.error("<== RangerServiceKafka.validateConfig Error:" + e); + throw e; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceKafka.validateConfig Response : (" + ret + + " )"); + } + return ret; + } + + @Override + public List<String> lookupResource(ResourceLookupContext context) + throws Exception { + + ServiceKafkaClient serviceKafkaClient = ServiceKafkaConnectionMgr + .getKafkaClient(serviceName, configs); + return serviceKafkaClient.getResources(context); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java ---------------------------------------------------------------------- diff --git a/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java new file mode 100644 index 0000000..a62bd95 --- /dev/null +++ b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.kafka.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import kafka.utils.ZkUtils; + +import org.I0Itec.zkclient.ZkClient; +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.client.BaseClient; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.plugin.util.TimedEventUtil; + +import scala.collection.Iterator; +import scala.collection.Seq; + +public class ServiceKafkaClient { + public static final Logger LOG = Logger.getLogger(ServiceKafkaClient.class); + + enum RESOURCE_TYPE { + TOPIC + } + + String serviceName = null; + String zookeeperConnect = null; + private static final String errMessage = " You can still save the repository and start creating " + + "policies, but you would not be able to use autocomplete for " + + "resource names. Check server logs for more info."; + + private static final String TOPIC_KEY = "topic"; + private static final long LOOKUP_TIMEOUT_SEC = 5; + + public ServiceKafkaClient(String serviceName, String zookeeperConnect) { + this.serviceName = serviceName; + this.zookeeperConnect = zookeeperConnect; + } + + public HashMap<String, Object> testConnection() throws Exception { + String errMsg = errMessage; + boolean connectivityStatus = false; + HashMap<String, Object> responseData = new HashMap<String, Object>(); + try { + getTopicList(null); + // If it doesn't throw exception, then assume the instance is + // reachable + String successMsg = "TestConnection Successful"; + BaseClient.generateResponseDataMap(connectivityStatus, successMsg, + successMsg, null, null, responseData); + } catch (IOException e) { + LOG.error("Error connecting to Kafka. kafkaClient=" + this, e); + String failureMsg = "Unable to connect to Kafka instance." + + e.getMessage(); + BaseClient.generateResponseDataMap(connectivityStatus, failureMsg, + failureMsg + errMsg, null, null, responseData); + } + return responseData; + } + + public List<String> getTopicList(List<String> ignoreTopicList) + throws Exception { + + List<String> list = new ArrayList<String>(); + + ZkClient zkClient = new ZkClient(zookeeperConnect); + try { + Seq<String> topicList = ZkUtils.getChildrenParentMayNotExist( + zkClient, ZkUtils.BrokerTopicsPath()); + + Iterator<String> iter = topicList.iterator(); + while (iter.hasNext()) { + String topic = iter.next(); + if (ignoreTopicList == null || !ignoreTopicList.contains(topic)) { + list.add(topic); + } + } + } finally { + try { + zkClient.close(); + } catch (Exception ex) { + LOG.error("Error closing zookeeper", ex); + } + } + return list; + } + + /** + * @param serviceName + * @param context + * @return + */ + public List<String> getResources(ResourceLookupContext context) { + + String userInput = context.getUserInput(); + String resource = context.getResourceName(); + Map<String, List<String>> resourceMap = context.getResources(); + List<String> resultList = null; + List<String> topicList = null; + + RESOURCE_TYPE lookupResource = RESOURCE_TYPE.TOPIC; + + if (LOG.isDebugEnabled()) { + LOG.debug("<== getResources() UserInput: \"" + userInput + + "\" resource : " + resource + " resourceMap: " + + resourceMap); + } + + if (userInput != null && resource != null) { + if (resourceMap != null && !resourceMap.isEmpty()) { + topicList = resourceMap.get(TOPIC_KEY); + } + switch (resource.trim().toLowerCase()) { + case TOPIC_KEY: + lookupResource = RESOURCE_TYPE.TOPIC; + break; + default: + break; + } + } + + if (userInput != null) { + try { + Callable<List<String>> callableObj = null; + final String userInputFinal = userInput; + + final List<String> finalTopicList = topicList; + + if (lookupResource == RESOURCE_TYPE.TOPIC) { + // get the topic list for given Input + callableObj = new Callable<List<String>>() { + @Override + public List<String> call() { + List<String> retList = new ArrayList<String>(); + try { + List<String> list = getTopicList(finalTopicList); + if (userInputFinal != null + && !userInputFinal.isEmpty()) { + for (String value : list) { + if (value.startsWith(userInputFinal)) { + retList.add(value); + } + } + } else { + retList.addAll(list); + } + } catch (Exception ex) { + LOG.error("Error getting topic.", ex); + } + return retList; + }; + }; + } + // If we need to do lookup + if (callableObj != null) { + synchronized (this) { + resultList = TimedEventUtil.timedTask(callableObj, + LOOKUP_TIMEOUT_SEC, TimeUnit.SECONDS); + } + } + } catch (Exception e) { + LOG.error("Unable to get hive resources.", e); + } + } + + return resultList; + } + + @Override + public String toString() { + return "ServiceKafkaClient [serviceName=" + serviceName + + ", zookeeperConnect=" + zookeeperConnect + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java ---------------------------------------------------------------------- diff --git a/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java new file mode 100644 index 0000000..a6d9d55 --- /dev/null +++ b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.kafka.client; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.log4j.Logger; + +public class ServiceKafkaConnectionMgr { + public static final Logger LOG = Logger + .getLogger(ServiceKafkaConnectionMgr.class); + + static public ServiceKafkaClient getKafkaClient(String serviceName, + Map<String, String> configs) throws Exception { + String zookeeperConnect = configs.get("zookeeper.connect"); + if (zookeeperConnect != null) { + ServiceKafkaClient serviceKafkaClient = new ServiceKafkaClient( + serviceName, zookeeperConnect); + return serviceKafkaClient; + } + throw new Exception("Required properties are not set for " + + serviceName + ". URL or Zookeeper information not provided."); + } + + /** + * @param serviceName + * @param configs + * @return + */ + public static HashMap<String, Object> testConnection(String serviceName, + Map<String, String> configs) throws Exception { + ServiceKafkaClient serviceKafkaClient = getKafkaClient(serviceName, + configs); + return serviceKafkaClient.testConnection(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java ---------------------------------------------------------------------- diff --git a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java index 3696299..2e6d0ac 100644 --- a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java +++ b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java @@ -196,7 +196,7 @@ public class ServiceSolrClient { RESOURCE_TYPE lookupResource = RESOURCE_TYPE.COLLECTION; if (LOG.isDebugEnabled()) { - LOG.debug("<== HiveResourceMgr.getHiveResources() UserInput: \"" + LOG.debug("<== getResources() UserInput: \"" + userInput + "\" resource : " + resource + " resourceMap: " + resourceMap); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8845eed..09899c6 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,7 @@ <module>hive-agent</module> <module>knox-agent</module> <module>storm-agent</module> + <module>plugin-kafka</module> <module>plugin-solr</module> <module>plugin-yarn</module> <module>ranger_solrj</module> @@ -364,6 +365,7 @@ <descriptor>src/main/assembly/hbase-agent.xml</descriptor> <descriptor>src/main/assembly/knox-agent.xml</descriptor> <descriptor>src/main/assembly/storm-agent.xml</descriptor> + <descriptor>src/main/assembly/plugin-kafka.xml</descriptor> <descriptor>src/main/assembly/plugin-yarn.xml</descriptor> <descriptor>src/main/assembly/plugin-solr.xml</descriptor> <descriptor>src/main/assembly/admin-web.xml</descriptor> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/src/main/assembly/plugin-kafka.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/plugin-kafka.xml b/src/main/assembly/plugin-kafka.xml new file mode 100644 index 0000000..b61039d --- /dev/null +++ b/src/main/assembly/plugin-kafka.xml @@ -0,0 +1,157 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>kafka-plugin</id> + <formats> + <format>tar.gz</format> + <format>zip</format> + </formats> + <baseDirectory>${project.name}-${project.version}-kafka-plugin</baseDirectory> + <includeBaseDirectory>true</includeBaseDirectory> + <moduleSets> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <outputDirectory>/lib</outputDirectory> + <unpack>false</unpack> + <includes> + <include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include> + <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-common-plus:jar:${hadoop-common.version}</include> + <include>com.google.code.gson:gson</include> + <include>org.eclipse.persistence:eclipselink</include> + <include>org.eclipse.persistence:javax.persistence</include> + <include>commons-collections:commons-collections</include> + <include>com.sun.jersey:jersey-bundle</include> + <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> + <include>com.google.guava:guava:jar:${guava.version}</include> + <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include> + <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include> + <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include> + <include>org.noggit:noggit:jar:${noggit.version}</include> + </includes> + </dependencySet> + <dependencySet> + <outputDirectory>/install/lib</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <includes> + <include>commons-cli:commons-cli</include> + <include>commons-collections:commons-collections</include> + <include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include> + <include>commons-io:commons-io:jar:${commons.io.version}</include> + <include>commons-lang:commons-lang:jar:${commons.lang.version}</include> + <include>commons-logging:commons-logging</include> + <include>com.google.guava:guava:jar:${guava.version}</include> + <include>org.hamcrest:hamcrest-all</include> + <include>junit:junit</include> + <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> + <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-auth:jar:${hadoop-common.version}</include> + <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include> + <include>org.apache.ranger:credentialbuilder</include> + </includes> + </dependencySet> + </dependencySets> + <outputDirectory>/lib</outputDirectory> + </binaries> + <includes> + <include>org.apache.ranger:ranger_solrj</include> + <include>security_plugins.ranger-plugins-audit:ranger-plugins-audit</include> + <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include> + <include>security_plugins.ranger-plugins-impl:ranger-plugins-impl</include> + <include>security_plugins.ranger-plugins-common:ranger-plugins-common</include> + <include>security_plugins.ranger-kafka-plugin:ranger-kafka-plugin</include> + </includes> + </moduleSet> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <outputDirectory>/install/lib</outputDirectory> + <unpack>false</unpack> + </binaries> + <includes> + <include>security_plugins.ranger-plugins-installer:ranger-plugins-installer</include> + <include>org.apache.ranger:credentialbuilder</include> + </includes> + </moduleSet> + </moduleSets> + <fileSets> + <!-- conf.templates for enable --> + <fileSet> + <outputDirectory>/install/conf.templates/enable</outputDirectory> + <directory>plugin-kafka/conf</directory> + <excludes> + <exclude>*.sh</exclude> + </excludes> + <fileMode>700</fileMode> + </fileSet> + <fileSet> + <outputDirectory>/install/conf.templates/disable</outputDirectory> + <directory>plugin-kafka/disable-conf</directory> + <fileMode>700</fileMode> + </fileSet> + <fileSet> + <outputDirectory>/install/conf.templates/default</outputDirectory> + <directory>plugin-kafka/template</directory> + <fileMode>700</fileMode> + </fileSet> + <!-- version file --> + <fileSet> + <outputDirectory>/</outputDirectory> + <directory>${project.build.outputDirectory}</directory> + <includes> + <include>version</include> + </includes> + <fileMode>444</fileMode> + </fileSet> + </fileSets> + <!-- enable/disable script for Plugin --> + <files> + <file> + <source>agents-common/scripts/enable-agent.sh</source> + <outputDirectory>/</outputDirectory> + <destName>enable-kafka-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>agents-common/scripts/enable-agent.sh</source> + <outputDirectory>/</outputDirectory> + <destName>disable-kafka-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>plugin-kafka/scripts/install.properties</source> + <outputDirectory>/</outputDirectory> + <destName>install.properties</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>plugin-kafka/scripts/kafka-plugin-install.properties</source> + <outputDirectory>/</outputDirectory> + <destName>kafka-plugin-install.properties</destName> + <fileMode>755</fileMode> + </file> + </files> +</assembly>
