RMP-3395: Pluggable authorization API for KMS Signed-off-by: Velmurugan Periasamy <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/75783f72 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/75783f72 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/75783f72 Branch: refs/heads/master Commit: 75783f728178f227ac5d6252e6b7d2e1e6ec02a3 Parents: 52ae981 Author: Gautam Borad <[email protected]> Authored: Tue Apr 7 13:26:17 2015 +0530 Committer: Velmurugan Periasamy <[email protected]> Committed: Tue Apr 7 10:44:06 2015 -0400 ---------------------------------------------------------------------- .../plugin/store/EmbeddedServiceDefsUtil.java | 10 +- .../service-defs/ranger-servicedef-kms.json | 109 ++++++ kms/config/kms-webapp/dbks-site.xml | 12 +- kms/config/kms-webapp/kms-site.xml | 2 +- .../crypto/key/kms/server/KMSConfiguration.java | 2 +- plugin-kms/.gitignore | 1 + plugin-kms/bin/.gitignore | 1 + plugin-kms/conf/ranger-kms-audit-changes.cfg | 45 +++ plugin-kms/conf/ranger-kms-audit.xml | 233 ++++++++++++ plugin-kms/conf/ranger-kms-security-changes.cfg | 26 ++ plugin-kms/conf/ranger-kms-security.xml | 67 ++++ .../conf/ranger-policymgr-ssl-changes.cfg | 23 ++ plugin-kms/conf/ranger-policymgr-ssl.xml | 63 ++++ plugin-kms/pom.xml | 51 +++ plugin-kms/scripts/install.properties | 112 ++++++ plugin-kms/scripts/install.sh | 364 +++++++++++++++++++ .../scripts/kms-plugin-install.properties | 23 ++ plugin-kms/scripts/uninstall.sh | 70 ++++ .../kms/authorizer/RangerKmsAuthorizer.java | 330 +++++++++++++++++ .../ranger/services/kms/RangerServiceKMS.java | 73 ++++ .../ranger/services/kms/client/KMSClient.java | 281 ++++++++++++++ .../services/kms/client/KMSConnectionMgr.java | 43 +++ .../services/kms/client/KMSResourceMgr.java | 88 +++++ .../client/json/model/KMSSchedulerResponse.java | 105 ++++++ pom.xml | 2 + src/main/assembly/plugin-kms.xml | 150 ++++++++ 26 files changed, 2281 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java old mode 100644 new mode 100755 index 75cf905..2930606 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -49,6 +49,7 @@ public class EmbeddedServiceDefsUtil { public static final String EMBEDDED_SERVICEDEF_KNOX_NAME = "knox"; public static final String EMBEDDED_SERVICEDEF_STORM_NAME = "storm"; public static final String EMBEDDED_SERVICEDEF_YARN_NAME = "yarn"; + public static final String EMBEDDED_SERVICEDEF_KMS_NAME = "kms"; public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; private static EmbeddedServiceDefsUtil instance = new EmbeddedServiceDefsUtil(); @@ -60,6 +61,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef knoxServiceDef = null; private RangerServiceDef stormServiceDef = null; private RangerServiceDef yarnServiceDef = null; + private RangerServiceDef kmsServiceDef = null; private Gson gsonBuilder = null; @@ -86,6 +88,7 @@ public class EmbeddedServiceDefsUtil { knoxServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KNOX_NAME); stormServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_STORM_NAME); yarnServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_YARN_NAME); + kmsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KMS_NAME); } catch(Throwable excp) { LOG.fatal("EmbeddedServiceDefsUtil.init(): failed", excp); } @@ -116,8 +119,11 @@ public class EmbeddedServiceDefsUtil { public long getYarnServiceDefId() { return getId(yarnServiceDef); } - - + + public long getKmsServiceDefId() { + return getId(kmsServiceDef); + } + private long getId(RangerServiceDef serviceDef) { return serviceDef == null || serviceDef.getId() == null ? -1 : serviceDef.getId().longValue(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/agents-common/src/main/resources/service-defs/ranger-servicedef-kms.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-kms.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-kms.json new file mode 100755 index 0000000..2ca22f4 --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-kms.json @@ -0,0 +1,109 @@ +{ + "name": "kms", + "implClass": "org.apache.ranger.services.kms.RangerServiceKMS", + "label": "KMS", + "description": "KMS", + "createTime": "20150329-11:03:47.095--0800", + "updateTime": "20150329-11:03:47.095--0800", + "version": 1, + "resources": + [ + { + "name": "keyname", + "type": "string", + "level": 1, + "mandatory": false, + "lookupSupported": true, + "recursiveSupported": true, + "label": "Key Name", + "description": "Key Name" + } + ], + + "accessTypes": + [ + { + "name": "create", + "label": "Create" + }, + + { + "name": "delete", + "label": "Delete" + }, + + { + "name": "rollover", + "label": "Rollover" + }, + + { + "name": "setkeymaterial", + "label": "Set Key Material" + }, + + { + "name": "get", + "label": "Get" + }, + + { + "name": "getkeys", + "label": "Get Keys" + }, + + { + "name": "getmetadata", + "label": "Get Metadata" + }, + + { + "name": "generateeek", + "label": "Generate EEK" + }, + + { + "name": "decrypteek", + "label": "Decrypt EEK" + } + ], + + "configs": + [ + { + "name": "provider", + "type": "string", + "mandatory": true, + "label": "KMS URL" + }, + + { + "name": "username", + "type": "string", + "mandatory": true, + "label": "Username" + }, + + { + "name": "password", + "type": "password", + "mandatory": true, + "label": "Password" + } + ], + + "enums": + [ + + ], + + "contextEnrichers": + [ + + ], + + "policyConditions": + [ + + ] +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/kms/config/kms-webapp/dbks-site.xml ---------------------------------------------------------------------- diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml index a428c6f..00bdd3e 100644 --- a/kms/config/kms-webapp/dbks-site.xml +++ b/kms/config/kms-webapp/dbks-site.xml @@ -1,6 +1,16 @@ <?xml version="1.0" encoding="UTF-8"?> <configuration> + <!-- Blacklist for authorization --> + + <!--<property> + <name>hadoop.kms.blacklist.CREATE</name> + <value>hdfs</value> + <description> + Blacklist for create-key operations. + </description> + </property>--> + <!-- Encryption key Password --> <property> @@ -47,7 +57,7 @@ <property> <name>ranger.db.root.password</name> - <value>root</value> + <value></value> <description> Database root user name used for operation </description> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/kms/config/kms-webapp/kms-site.xml ---------------------------------------------------------------------- diff --git a/kms/config/kms-webapp/kms-site.xml b/kms/config/kms-webapp/kms-site.xml index 06e7ec4..b61d1b2 100644 --- a/kms/config/kms-webapp/kms-site.xml +++ b/kms/config/kms-webapp/kms-site.xml @@ -172,7 +172,7 @@ <property> <name>hadoop.kms.security.authorization.manager</name> - <value></value> + <value>org.apache.ranger.authorization.kms.authorizer.RangerKmsAuthorizer</value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java index d6c77ea..cc5c202 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java @@ -33,7 +33,7 @@ public class KMSConfiguration { public static final String KMS_CONFIG_DIR = "kms.config.dir"; public static final String KMS_SITE_XML = "kms-site.xml"; - public static final String KMS_ACLS_XML = "kms-acls.xml"; + public static final String KMS_ACLS_XML = "dbks-site.xml"; public static final String CONFIG_PREFIX = "hadoop.kms."; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/.gitignore ---------------------------------------------------------------------- diff --git a/plugin-kms/.gitignore b/plugin-kms/.gitignore new file mode 100755 index 0000000..ea8c4bf --- /dev/null +++ b/plugin-kms/.gitignore @@ -0,0 +1 @@ +/target http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/bin/.gitignore ---------------------------------------------------------------------- diff --git a/plugin-kms/bin/.gitignore b/plugin-kms/bin/.gitignore new file mode 100755 index 0000000..ae3c172 --- /dev/null +++ b/plugin-kms/bin/.gitignore @@ -0,0 +1 @@ +/bin/ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/conf/ranger-kms-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-kms/conf/ranger-kms-audit-changes.cfg b/plugin-kms/conf/ranger-kms-audit-changes.cfg new file mode 100644 index 0000000..2d6d414 --- /dev/null +++ b/plugin-kms/conf/ranger-kms-audit-changes.cfg @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT_DB_JDBC_URL% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists +xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT_DB_JDBC_DRIVER% mod create-if-not-exists + +xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.directory %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.directory %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/conf/ranger-kms-audit.xml ---------------------------------------------------------------------- diff --git a/plugin-kms/conf/ranger-kms-audit.xml b/plugin-kms/conf/ranger-kms-audit.xml new file mode 100755 index 0000000..f1c4187 --- /dev/null +++ b/plugin-kms/conf/ranger-kms-audit.xml @@ -0,0 +1,233 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>xasecure.audit.is.enabled</name> + <value>true</value> + </property> + + <!-- DB audit provider configuration --> + <property> + <name>xasecure.audit.db.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.db.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.db.batch.size</name> + <value>100</value> + </property> + + <!-- Properties whose name begin with "xasecure.audit.jpa." are used to configure JPA --> + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.url</name> + <value>jdbc:mysql://localhost:3306/ranger_audit</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.user</name> + <value>rangerlogger</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.password</name> + <value>none</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.driver</name> + <value>com.mysql.jdbc.Driver</value> + </property> + + <property> + <name>xasecure.audit.credential.provider.file</name> + <value>jceks://file/etc/ranger/kmsdev/auditcred.jceks</value> + </property> + + + <!-- HDFS audit provider configuration --> + <property> + <name>xasecure.audit.hdfs.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.hdfs.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.queue.size</name> + <value>1048576</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.encoding</name> + <value></value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.directory</name> + <value>hdfs://NAMENODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.file</name> + <value>%hostname%-audit.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name> + <value>900</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name> + <value>86400</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.directory</name> + <value>/var/log/kms/audit</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file</name> + <value>%time:yyyyMMdd-HHmm.ss%.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file.buffer.size.bytes</name> + <value>8192</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name> + <value>600</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.directory</name> + <value>/var/log/kms/audit/archive</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name> + <value>10</value> + </property> + + + <!-- Log4j audit provider configuration --> + <property> + <name>xasecure.audit.log4j.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.is.async</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.solr.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.solr.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.solr.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/conf/ranger-kms-security-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-kms/conf/ranger-kms-security-changes.cfg b/plugin-kms/conf/ranger-kms-security-changes.cfg new file mode 100644 index 0000000..fdd1723 --- /dev/null +++ b/plugin-kms/conf/ranger-kms-security-changes.cfg @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Change the original policy parameter to work with policy manager based. +# +# +ranger.plugin.kms.service.name %REPOSITORY_NAME% mod create-if-not-exists + +ranger.plugin.kms.policy.source.impl org.apache.ranger.admin.client.RangerAdminRESTClient mod create-if-not-exists + +ranger.plugin.kms.policy.rest.url %POLICY_MGR_URL% mod create-if-not-exists +ranger.plugin.kms.policy.rest.ssl.config.file /etc/kms/conf/ranger-policymgr-ssl.xml mod create-if-not-exists +ranger.plugin.kms.policy.pollIntervalMs 30000 mod create-if-not-exists +ranger.plugin.kms.policy.cache.dir %POLICY_CACHE_FILE_PATH% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/conf/ranger-kms-security.xml ---------------------------------------------------------------------- diff --git a/plugin-kms/conf/ranger-kms-security.xml b/plugin-kms/conf/ranger-kms-security.xml new file mode 100755 index 0000000..3aae5f1 --- /dev/null +++ b/plugin-kms/conf/ranger-kms-security.xml @@ -0,0 +1,67 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>ranger.plugin.kms.service.name</name> + <value>kmsdev</value> + <description> + Name of the Ranger service containing policies for this kms instance + </description> + </property> + + <property> + <name>ranger.plugin.kms.policy.source.impl</name> + <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value> + <description> + Class to retrieve policies from the source + </description> + </property> + + <property> + <name>ranger.plugin.kms.policy.rest.url</name> + <value>http://policymanagerhost:port</value> + <description> + URL to Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.kms.policy.rest.ssl.config.file</name> + <value>/etc/kms/conf/ranger-policymgr-ssl.xml</value> + <description> + Path to the file containing SSL details to contact Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.kms.policy.pollIntervalMs</name> + <value>30000</value> + <description> + How often to poll for changes in policies? + </description> + </property> + + <property> + <name>ranger.plugin.kms.policy.cache.dir</name> + <value>/etc/ranger/kmsdev/policycache</value> + <description> + Directory where Ranger policies are cached after successful retrieval from the source + </description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/conf/ranger-policymgr-ssl-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-kms/conf/ranger-policymgr-ssl-changes.cfg b/plugin-kms/conf/ranger-policymgr-ssl-changes.cfg new file mode 100644 index 0000000..6bf4265 --- /dev/null +++ b/plugin-kms/conf/ranger-policymgr-ssl-changes.cfg @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SSL Params +# +xasecure.policymgr.clientssl.keystore %SSL_KEYSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.keystore.password %SSL_KEYSTORE_PASSWORD% mod create-if-not-exists +xasecure.policymgr.clientssl.keystore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore %SSL_TRUSTSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore.password %SSL_TRUSTSTORE_PASSWORD% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/conf/ranger-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/plugin-kms/conf/ranger-policymgr-ssl.xml b/plugin-kms/conf/ranger-policymgr-ssl.xml new file mode 100755 index 0000000..6a9593f --- /dev/null +++ b/plugin-kms/conf/ranger-policymgr-ssl.xml @@ -0,0 +1,63 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <!-- The following properties are used for 2-way SSL client server validation --> + <property> + <name>xasecure.policymgr.clientssl.keystore</name> + <value>kmsdev-clientcert.jks</value> + <description> + Java Keystore files + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.password</name> + <value>none</value> + <description> + password for keystore + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore</name> + <value>cacerts-xasecure.jks</value> + <description> + java truststore file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.password</name> + <value>none</value> + <description> + java truststore password + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.credential.file</name> + <value>jceks://file/tmp/keystore-kmsdev-ssl.jceks</value> + <description> + java keystore credential file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.credential.file</name> + <value>jceks://file/tmp/truststore-kmsdev-ssl.jceks</value> + <description> + java truststore credential file + </description> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/pom.xml ---------------------------------------------------------------------- diff --git a/plugin-kms/pom.xml b/plugin-kms/pom.xml new file mode 100755 index 0000000..ddfde79 --- /dev/null +++ b/plugin-kms/pom.xml @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>security_plugins.ranger-kms-plugin</groupId> + <artifactId>ranger-kms-plugin</artifactId> + <name>KMS Security Plugin</name> + <description>KMS Security Plugin</description> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <parent> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger</artifactId> + <version>0.5.0</version> + <relativePath>..</relativePath> + </parent> + <dependencies> + <dependency> + <groupId>security_plugins.ranger-plugins-common</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>security_plugins.ranger-plugins-audit</groupId> + <artifactId>ranger-plugins-audit</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-kms</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/scripts/install.properties ---------------------------------------------------------------------- diff --git a/plugin-kms/scripts/install.properties b/plugin-kms/scripts/install.properties new file mode 100755 index 0000000..93cf4a1 --- /dev/null +++ b/plugin-kms/scripts/install.properties @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=kmsdev +# +REPOSITORY_NAME= + +# +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=false +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME= +XAAUDIT.DB.DATABASE_NAME= +XAAUDIT.DB.USER_NAME= +XAAUDIT.DB.PASSWORD= + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/kms/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/kms/audit/archive +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/kms/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/kms/audit/archive +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/kms/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/kms/audit/archive + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/kms/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/kms/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/kms/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/kms/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/scripts/install.sh ---------------------------------------------------------------------- diff --git a/plugin-kms/scripts/install.sh b/plugin-kms/scripts/install.sh new file mode 100755 index 0000000..d88dadf --- /dev/null +++ b/plugin-kms/scripts/install.sh @@ -0,0 +1,364 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +function create_jceks() +{ + +alias=$1 +pass=$2 +jceksFile=$3 + +ret=`hadoop credential create ${alias} -value ${pass} -provider jceks://file${jceksFile} 2>&1` +res=`echo $ret | grep 'already exist'` + +if ! [ "${res}" == "" ] +then + echo "Credential file already exists,recreating the file..." + hadoop credential delete ${alias} -provider jceks://file${jceksFile} + hadoop credential create ${alias} -value ${pass} -provider jceks://file${jceksFile} +fi +} + +#Update Properties to File +#$1 -> propertyName $2 -> newPropertyValue $3 -> fileName +updatePropertyToFile(){ + sed -i 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 + #validate=`sed -i 's/^'$1'=[^ ]*$/'$1'='$2'/g' $3` #for validation + validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation + #echo 'V1:'$validate + if test -z "$validate" ; then echo "[E] '$1' not found in $3 file while Updating....!!"; exit 1; fi + echo "[I] File $3 Updated successfully : {'$1'}" +} + +kms_dir=/usr/hdp/current/kms +kms_lib_dir=${kms_dir}/lib +kms_conf_dir=/etc/kms/conf + +hdp_dir=/usr/hdp/current/hadoop +hdp_lib_dir=${hdp_dir}/lib +hdp_conf_dir=/etc/hadoop/conf + +export CONFIG_FILE_OWNER="kms:hadoop" + + +if [ ! -d "${hdp_dir}" ] +then + echo "ERROR: Invalid HADOOP HOME Directory: [${hdp_dir}]. Exiting ..." + exit 1 +fi + +#echo "Hadoop Configuration Path: ${hdp_conf_dir}" + +if [ ! -f ${hdp_conf_dir}/hadoop-env.sh ] +then + echo "ERROR: Invalid HADOOP CONF Directory: [${hdp_conf_dir}]." + echo "ERROR: Unable to locate: hadoop-env.sh. Exiting ..." + exit 1 +fi + +install_dir=`dirname $0` + +[ "${install_dir}" = "." ] && install_dir=`pwd` + +#echo "Current Install Directory: [${install_dir}]" + +#verify sql-connector path is valid +SQL_CONNECTOR_JAR=`grep '^SQL_CONNECTOR_JAR' ${install_dir}/install.properties | awk -F= '{ print $2 }'` +echo "[I] Checking SQL CONNECTOR FILE : $SQL_CONNECTOR_JAR" +if test -f "$SQL_CONNECTOR_JAR"; then + echo "[I] SQL CONNECTOR FILE : $SQL_CONNECTOR_JAR file found" +else + echo "[E] SQL CONNECTOR FILE : $SQL_CONNECTOR_JAR does not exists" ; exit 1; +fi +#copying sql connector jar file to lib directory +cp $SQL_CONNECTOR_JAR ${install_dir}/lib + + +# +# --- Backup current configuration for backup - START +# + +COMPONENT_NAME=kms + +XASECURE_VERSION=`cat ${install_dir}/version` + +CFG_DIR=${kms_conf_dir} +XASECURE_ROOT=/etc/xasecure/${COMPONENT_NAME} +BACKUP_TYPE=pre +CUR_VERSION_FILE=${XASECURE_ROOT}/.current_version +CUR_CFG_DIR_FILE=${XASECURE_ROOT}/.config_dir +PRE_INSTALL_CONFIG=${XASECURE_ROOT}/${BACKUP_TYPE}-${XASECURE_VERSION} + +backup_dt=`date '+%Y%m%d%H%M%S'` + +if [ -d "${PRE_INSTALL_CONFIG}" ] +then + PRE_INSTALL_CONFIG="${PRE_INSTALL_CONFIG}.${backup_dt}" +fi + +if [ -d ${CFG_DIR} ] +then + ( cd ${CFG_DIR} ; find . -print | cpio -pdm ${PRE_INSTALL_CONFIG} ) + [ -f ${CUR_VERSION_FILE} ] && mv ${CUR_VERSION_FILE} ${CUR_VERSION_FILE}-${backup_dt} + echo ${XASECURE_VERSION} > ${CUR_VERSION_FILE} + echo ${CFG_DIR} > ${CUR_CFG_DIR_FILE} +else + echo "ERROR: Unable to find configuration directory: [${CFG_DIR}]" + exit 1 +fi + +cp -f ${install_dir}/uninstall.sh ${XASECURE_ROOT}/ + +# +# --- Backup current configuration for backup - END + +dt=`date '+%Y%m%d%H%M%S'` +for f in ${install_dir}/conf/* +do + if [ -f ${f} ] + then + fn=`basename $f` + if [ ! -f ${kms_conf_dir}/${fn} ] + then + echo "+cp ${f} ${kms_conf_dir}/${fn}" + cp ${f} ${kms_conf_dir}/${fn} + else + echo "WARN: ${fn} already exists in the ${kms_conf_dir} - Using existing configuration ${fn}" + fi + fi +done + +#echo "Hadoop XASecure Library Path: ${hdp_lib_dir}" + +if [ ! -d ${kms_lib_dir} ] +then + echo "+mkdir -p ${kms_lib_dir}" + mkdir -p ${kms_lib_dir} +fi + +for f in ${install_dir}/dist/*.jar +do + if [ -f ${f} ] + then + fn=`basename $f` + echo "+cp ${f} ${kms_lib_dir}/${fn}" + cp ${f} ${kms_lib_dir}/${fn} + fi +done + + +for f in ${install_dir}/dist/*.jar +do + if [ -f ${f} ] + then + fn=`basename $f` + echo "+cp ${f} ${kms_lib_dir}/${fn}" + cp ${f} ${kms_lib_dir}/${fn} + fi +done + +if [ -d ${install_dir}/lib ] +then + for f in ${install_dir}/lib/*.jar + do + if [ -f ${f} ] + then + fn=`basename $f` + if [ -f ${kms_lib_dir}/${fn} ] + then + cdt=`date '+%s'` + echo "+mv ${kms_lib_dir}/${fn} ${kms_lib_dir}/.${fn}.${cdt}" + mv ${kms_lib_dir}/${fn} ${kms_lib_dir}/.${fn}.${cdt} + fi + echo "+cp ${f} ${kms_lib_dir}/${fn}" + cp ${f} ${kms_lib_dir}/${fn} + fi + done +fi + + +CredFile=`grep '^CREDENTIAL_PROVIDER_FILE' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + +if ! [ `echo ${CredFile} | grep '^/.*'` ] +then + echo "ERROR:Please enter the Credential File Store with proper file path" + exit 1 +fi + +dirno=`echo ${CredFile}| awk -F"/" '{ print NF}'` + +if [ ${dirno} -gt 2 ]; +then + pardir=`echo ${CredFile} | awk -F'/[^/]*$' '{ print $1 }'` + if [ ! -d ${pardir} ]; + then + mkdir -p ${pardir} + if [ $? -eq 0 ]; + then + chmod go+rx ${pardir} + else + echo "ERROR: Unable to create credential store file path" + fi + fi +fi + + +# +# Generate Credential Provider file and Credential for Audit DB access. +# + + +auditCredAlias="auditDBCred" + +auditdbCred=`grep '^XAAUDIT.DB.PASSWORD' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + +create_jceks ${auditCredAlias} ${auditdbCred} ${CredFile} + + +# +# Generate Credential Provider file and Credential for SSL KEYSTORE AND TRUSTSTORE +# + + +sslkeystoreAlias="sslKeyStore" + +sslkeystoreCred=`head -1 /etc/xasecure/ssl/certs/${repoName}.maze` + +create_jceks ${sslkeystoreAlias} ${sslkeystoreCred} ${CredFile} + + + +ssltruststoreAlias="sslTrustStore" + +ssltruststoreCred=`grep '^SSL_TRUSTSTORE_PASSWORD' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + +create_jceks ${ssltruststoreAlias} ${ssltruststoreCred} ${CredFile} + +chmod go+rx ${pardir} +chmod go+r ${CredFile} +chown ${CONFIG_FILE_OWNER} ${CredFile} + +PROP_ARGS="-p ${install_dir}/install.properties" +to_file="${install_dir}/install.properties" +DB_FLAVOR=`grep '^XAAUDIT.DB.FLAVOUR' ${install_dir}/install.properties | awk -F= '{ print $2 }'` +DB_FLAVOR=`echo $DB_FLAVOR | tr '[:lower:]' '[:upper:]'` +if [ "${DB_FLAVOR}" == "" ] +then + $DB_FLAVOR="MYSQL" +fi +echo "[I] Updating install.properites setting for : $DB_FLAVOR" + +if [ "${DB_FLAVOR}" == "MYSQL" ] +then + audit_db_hostname=`grep '^XAAUDIT.DB.HOSTNAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + audit_db_name=`grep '^XAAUDIT.DB.DATABASE_NAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + propertyName=XAAUDIT.DB.JDBC_URL + newPropertyValue="jdbc:mysql://${audit_db_hostname}/${audit_db_name}" + updatePropertyToFile $propertyName $newPropertyValue $to_file + + propertyName=XAAUDIT.DB.JDBC_DRIVER + newPropertyValue="com.mysql.jdbc.Driver" + updatePropertyToFile $propertyName $newPropertyValue $to_file +fi + +if [ "${DB_FLAVOR}" == "ORACLE" ] +then + audit_db_hostname=`grep '^XAAUDIT.DB.HOSTNAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + propertyName=XAAUDIT.DB.JDBC_URL + newPropertyValue="jdbc:oracle:thin:\@//${audit_db_hostname}" + updatePropertyToFile $propertyName $newPropertyValue $to_file + + propertyName=XAAUDIT.DB.JDBC_DRIVER + newPropertyValue="oracle.jdbc.OracleDriver" + updatePropertyToFile $propertyName $newPropertyValue $to_file +fi + +if [ "${DB_FLAVOR}" == "POSTGRES" ] +then + audit_db_hostname=`grep '^XAAUDIT.DB.HOSTNAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + audit_db_name=`grep '^XAAUDIT.DB.DATABASE_NAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + propertyName=XAAUDIT.DB.JDBC_URL + newPropertyValue="jdbc:postgresql://${audit_db_hostname}/${audit_db_name}" + updatePropertyToFile $propertyName $newPropertyValue $to_file + propertyName=XAAUDIT.DB.JDBC_DRIVER + newPropertyValue="org.postgresql.Driver" + updatePropertyToFile $propertyName $newPropertyValue $to_file +fi +if [ "${DB_FLAVOR}" == "SQLSERVER" ] +then + audit_db_hostname=`grep '^XAAUDIT.DB.HOSTNAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + audit_db_name=`grep '^XAAUDIT.DB.DATABASE_NAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + propertyName=XAAUDIT.DB.JDBC_URL + newPropertyValue="jdbc:sqlserver://${audit_db_hostname};databaseName=${audit_db_name}" + updatePropertyToFile $propertyName $newPropertyValue $to_file + + propertyName=XAAUDIT.DB.JDBC_DRIVER + newPropertyValue="com.microsoft.sqlserver.jdbc.SQLServerDriver" + updatePropertyToFile $propertyName $newPropertyValue $to_file +fi +for f in ${install_dir}/installer/conf/*-changes.cfg +do + if [ -f ${f} ] + then + fn=`basename $f` + orgfn=`echo $fn | sed -e 's:-changes.cfg:.xml:'` + fullpathorgfn="${kms_conf_dir}/${orgfn}" + if [ ! -f ${fullpathorgfn} ] + then + echo "ERROR: Unable to find ${fullpathorgfn}" + exit 1 + fi + archivefn="${kms_conf_dir}/.${orgfn}.${dt}" + newfn="${kms_conf_dir}/.${orgfn}-new.${dt}" + cp ${fullpathorgfn} ${archivefn} + if [ $? -eq 0 ] + then + cp="${install_dir}/installer/lib/*:${hdp_dir}/*:${hdp_lib_dir}/*" + java -cp "${cp}" org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} ${PROP_ARGS} + if [ $? -eq 0 ] + then + diff -w ${newfn} ${fullpathorgfn} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + #echo "Changing config file: ${fullpathorgfn} with following changes:" + #echo "===============================================================" + #diff -w ${newfn} ${fullpathorgfn} + #echo "===============================================================" + echo "NOTE: Current config file: ${fullpathorgfn} is being saved as ${archivefn}" + #echo "===============================================================" + cp ${newfn} ${fullpathorgfn} + fi + else + echo "ERROR: Unable to make changes to config. file: ${fullpathorgfn}" + echo "exiting ...." + exit 1 + fi + else + echo "ERROR: Unable to save config. file: ${fullpathorgfn} to ${archivefn}" + echo "exiting ...." + exit 1 + fi + fi +done + +chmod go-rwx ${kms_conf_dir}/xasecure-policymgr-ssl.xml + +chown ${CONFIG_FILE_OWNER} ${kms_conf_dir}/xasecure-policymgr-ssl.xml + +exit 0 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/scripts/kms-plugin-install.properties ---------------------------------------------------------------------- diff --git a/plugin-kms/scripts/kms-plugin-install.properties b/plugin-kms/scripts/kms-plugin-install.properties new file mode 100755 index 0000000..b79d529 --- /dev/null +++ b/plugin-kms/scripts/kms-plugin-install.properties @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# installation properties for this plugin + +# +# Name of the directory where the component's lib and conf directory exist. +# This location should be relative to the parent of the directory containing +# the plugin installation files. +# +COMPONENT_INSTALL_DIR_NAME=ranger-kms http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/scripts/uninstall.sh ---------------------------------------------------------------------- diff --git a/plugin-kms/scripts/uninstall.sh b/plugin-kms/scripts/uninstall.sh new file mode 100755 index 0000000..e0d3584 --- /dev/null +++ b/plugin-kms/scripts/uninstall.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPONENT_NAME=hbase +CFG_DIR=/etc/${COMPONENT_NAME}/conf +XASECURE_ROOT=/etc/xasecure/${COMPONENT_NAME} +BACKUP_TYPE=pre +CUR_VERSION_FILE=${XASECURE_ROOT}/.current_version +CUR_CFG_DIR_FILE=${XASECURE_ROOT}/.config_dir +if [ -f ${CUR_VERSION_FILE} ] +then + XASECURE_VERSION=`cat ${CUR_VERSION_FILE}` + PRE_INSTALL_CONFIG=${XASECURE_ROOT}/${BACKUP_TYPE}-${XASECURE_VERSION} + dt=`date '+%Y%m%d%H%M%S'` + if [ -d "${PRE_INSTALL_CONFIG}" ] + then + if [ -f ${CUR_CFG_DIR_FILE} ] + then + CFG_DIR=`cat ${CUR_CFG_DIR_FILE}` + fi + [ -d ${CFG_DIR} ] && mv ${CFG_DIR} ${CFG_DIR}-${dt} + ( cd ${PRE_INSTALL_CONFIG} ; find . -print | cpio -pdm ${CFG_DIR} ) + [ -f ${CUR_VERSION_FILE} ] && mv ${CUR_VERSION_FILE} ${CUR_VERSION_FILE}-uninstalled-${dt} + echo "XASecure version - ${XASECURE_VERSION} has been uninstalled successfully." + else + echo "ERROR: Unable to find pre-install configuration directory: [${PRE_INSTALL_CONFIG}]" + exit 1 + fi +else + cd ${CFG_DIR} + saved_files=`find . -type f -name '.*' | sort | grep -v -- '-new.' | grep '[0-9]*$' | grep -v -- '-[0-9]*$' | sed -e 's:\.[0-9]*$::' | sed -e 's:^./::' | sort -u` + dt=`date '+%Y%m%d%H%M%S'` + if [ "${saved_files}" != "" ] + then + for f in ${saved_files} + do + oldf=`ls ${f}.[0-9]* | sort | head -1` + if [ -f "${oldf}" ] + then + nf=`echo ${f} | sed -e 's:^\.::'` + if [ -f "${nf}" ] + then + echo "+cp -p ${nf} .${nf}-${dt}" + cp -p ${nf} .${nf}-${dt} + echo "+cp ${oldf} ${nf}" + cp ${oldf} ${nf} + else + echo "ERROR: ${nf} not found to save. However, old file is being recovered." + echo "+cp -p ${oldf} ${nf}" + cp -p ${oldf} ${nf} + fi + fi + done + echo "XASecure configuration has been uninstalled successfully." + fi +fi http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java ---------------------------------------------------------------------- diff --git a/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java new file mode 100755 index 0000000..96fcf31 --- /dev/null +++ b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java @@ -0,0 +1,330 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.authorization.kms.authorizer; + +import java.net.InetAddress; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.kms.server.KMSACLsType; +import org.apache.hadoop.crypto.key.kms.server.KMSConfiguration; +import org.apache.hadoop.crypto.key.kms.server.KMSWebApp; +import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp; +import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Sets; + +public class RangerKmsAuthorizer implements Runnable, KeyACLs { + private static final Logger LOG = LoggerFactory.getLogger(RangerKmsAuthorizer.class); + + private static final String UNAUTHORIZED_MSG_WITH_KEY = + "User:%s not allowed to do '%s' on '%s'"; + + private static final String UNAUTHORIZED_MSG_WITHOUT_KEY = + "User:%s not allowed to do '%s'"; + + public static final int RELOADER_SLEEP_MILLIS = 1000; + + private volatile Map<Type, AccessControlList> blacklistedAcls; + + private long lastReload; + + private ScheduledExecutorService executorService; + + public static final String ACCESS_TYPE_DECRYPT_EEK = "decrypteek"; + public static final String ACCESS_TYPE_GENERATE_EEK = "generateeek"; + public static final String ACCESS_TYPE_GET_METADATA = "getmetadata"; + public static final String ACCESS_TYPE_GET_KEYS = "getkeys"; + public static final String ACCESS_TYPE_GET = "get"; + public static final String ACCESS_TYPE_SET_KEY_MATERIAL= "setkeymaterial"; + public static final String ACCESS_TYPE_ROLLOVER = "rollover"; + public static final String ACCESS_TYPE_CREATE = "create"; + public static final String ACCESS_TYPE_DELETE = "delete"; + + private static volatile RangerKMSPlugin kmsPlugin = null; + + RangerKmsAuthorizer(Configuration conf) { + if (conf == null) { + conf = loadACLs(); + } + setKMSACLs(conf); + init(conf); + } + + public RangerKmsAuthorizer() { + this(null); + } + + @Override + public void run() { + try { + if (KMSConfiguration.isACLsFileNewer(lastReload)) { + setKMSACLs(loadACLs()); + } + } catch (Exception ex) { + LOG.warn( + String.format("Could not reload ACLs file: '%s'", ex.toString()), ex); + } + } + + private Configuration loadACLs() { + LOG.debug("Loading ACLs file"); + lastReload = System.currentTimeMillis(); + Configuration conf = KMSConfiguration.getACLsConf(); + // triggering the resource loading. + conf.get(Type.CREATE.getAclConfigKey()); + return conf; + } + + public synchronized void startReloader() { + if (executorService == null) { + executorService = Executors.newScheduledThreadPool(1); + executorService.scheduleAtFixedRate(this, RELOADER_SLEEP_MILLIS, + RELOADER_SLEEP_MILLIS, TimeUnit.MILLISECONDS); + } + } + + public synchronized void stopReloader() { + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + } + + /** + * First Check if user is in ACL for the KMS operation, if yes, then + * return true if user is not present in any configured blacklist for + * the operation + * @param type KMS Operation + * @param ugi UserGroupInformation of user + * @return true is user has access + */ + @Override + public boolean hasAccess(Type type, UserGroupInformation ugi) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerKmsAuthorizer.hasAccess(" + type + ", " + ugi + ")"); + } + + boolean ret = false; + RangerKMSPlugin plugin = kmsPlugin; + String rangerAccessType = getRangerAccessType(type); + AccessControlList blacklist = blacklistedAcls.get(type); + ret = (blacklist == null) || !blacklist.isUserInList(ugi); + if(!ret){ + LOG.debug("Operation "+rangerAccessType+" blocked in the blacklist for user "+ugi.getUserName()); + } + + if(plugin != null && ret) { + RangerKMSAccessRequest request = new RangerKMSAccessRequest(rangerAccessType, ugi); + RangerAccessResult result = plugin.isAccessAllowed(request); + ret = result == null ? false : result.getIsAllowed(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerkmsAuthorizer.hasAccess(" + type + ", " + ugi + "): " + ret); + } + + return ret; + } + + @Override + public void assertAccess(Type aclType, UserGroupInformation ugi, KMSOp operation, String key) + throws AccessControlException { + if (!KMSWebApp.getACLs().hasAccess(aclType, ugi)) { + KMSWebApp.getUnauthorizedCallsMeter().mark(); + KMSWebApp.getKMSAudit().unauthorized(ugi, operation, key); + throw new AuthorizationException(String.format( + (key != null) ? UNAUTHORIZED_MSG_WITH_KEY + : UNAUTHORIZED_MSG_WITHOUT_KEY, + ugi.getShortUserName(), operation, key)); + } + } + + @Override + public boolean hasAccessToKey(String keyName, UserGroupInformation ugi, KeyOpType opType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerKmsAuthorizer.hasAccessToKey(" + keyName + ", " + ugi +", " + opType + ")"); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerKmsAuthorizer.hasAccessToKey(" + keyName + ", " + ugi +", " + opType + ")"); + } + + return true; + } + + @Override + public boolean isACLPresent(String keyName, KeyOpType opType) { + return true; + } + + @Override + public void startACLReloader() { + this.startReloader(); + } + + @Override + public void stopACLReloader() { + this.stopReloader(); + } + + public void init(Configuration conf) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerKmsAuthorizer.init()"); + } + + RangerKMSPlugin plugin = kmsPlugin; + + if(plugin == null) { + synchronized(RangerKmsAuthorizer.class) { + plugin = kmsPlugin; + + if(plugin == null) { + plugin = new RangerKMSPlugin(); + plugin.init(); + + kmsPlugin = plugin; + } + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerkmsAuthorizer.init()"); + } + } + + private void setKMSACLs(Configuration conf) { + Map<Type, AccessControlList> tempBlacklist = new HashMap<Type, AccessControlList>(); + for (Type aclType : Type.values()) { + String blacklistStr = conf.get(aclType.getBlacklistConfigKey()); + if (blacklistStr != null) { + // Only add if blacklist is present + tempBlacklist.put(aclType, new AccessControlList(blacklistStr)); + LOG.info("'{}' Blacklist '{}'", aclType, blacklistStr); + } + } + blacklistedAcls = tempBlacklist; + } + + private static String getRangerAccessType(KMSACLsType.Type accessType) { + String ret = null; + + switch(accessType) { + case CREATE: + ret = RangerKmsAuthorizer.ACCESS_TYPE_CREATE; + break; + + case DELETE: + ret = RangerKmsAuthorizer.ACCESS_TYPE_DELETE; + break; + + case ROLLOVER: + ret = RangerKmsAuthorizer.ACCESS_TYPE_ROLLOVER; + break; + + case GET: + ret = RangerKmsAuthorizer.ACCESS_TYPE_GET; + break; + + case GET_KEYS: + ret = RangerKmsAuthorizer.ACCESS_TYPE_GET_KEYS; + break; + + case GET_METADATA: + ret = RangerKmsAuthorizer.ACCESS_TYPE_GET_METADATA; + break; + + case SET_KEY_MATERIAL: + ret = RangerKmsAuthorizer.ACCESS_TYPE_SET_KEY_MATERIAL; + break; + + case GENERATE_EEK: + ret = RangerKmsAuthorizer.ACCESS_TYPE_GENERATE_EEK; + break; + + case DECRYPT_EEK: + ret = RangerKmsAuthorizer.ACCESS_TYPE_DECRYPT_EEK; + break; + } + return ret; + } + } + + class RangerKMSPlugin extends RangerBasePlugin { + public RangerKMSPlugin() { + super("kms", "kms"); + } + + @Override + public void init() { + super.init(); + + RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); + + super.setDefaultAuditHandler(auditHandler); + } + } + + class RangerKMSResource extends RangerAccessResourceImpl { + private static final String KEY_NAME = "keyname"; + + public RangerKMSResource(String keyname) { + setValue(KEY_NAME, keyname != null ? keyname : null); + } + } + + class RangerKMSAccessRequest extends RangerAccessRequestImpl { + public RangerKMSAccessRequest(String accessType, UserGroupInformation ugi) { + super.setResource(new RangerKMSResource("kms")); + super.setAccessType(accessType); + super.setUser(ugi.getShortUserName()); + super.setUserGroups(Sets.newHashSet(ugi.getGroupNames())); + super.setAccessTime(StringUtil.getUTCDate()); + super.setClientIPAddress(getRemoteIp()); + super.setAction(accessType); + } + + private static String getRemoteIp() { + String ret = null ; + InetAddress ip = Server.getRemoteIp() ; + if (ip != null) { + ret = ip.getHostAddress(); + } + return ret ; + } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/src/main/java/org/apache/ranger/services/kms/RangerServiceKMS.java ---------------------------------------------------------------------- diff --git a/plugin-kms/src/main/java/org/apache/ranger/services/kms/RangerServiceKMS.java b/plugin-kms/src/main/java/org/apache/ranger/services/kms/RangerServiceKMS.java new file mode 100755 index 0000000..8c3b7f3 --- /dev/null +++ b/plugin-kms/src/main/java/org/apache/ranger/services/kms/RangerServiceKMS.java @@ -0,0 +1,73 @@ +package org.apache.ranger.services.kms; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.services.kms.client.KMSResourceMgr; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class RangerServiceKMS extends RangerBaseService { + + private static final Log LOG = LogFactory.getLog(RangerServiceKMS.class); + + public RangerServiceKMS() { + super(); + } + + @Override + public void init(RangerServiceDef serviceDef, RangerService service) { + super.init(serviceDef, service); + } + + @Override + public HashMap<String,Object> validateConfig() throws Exception { + HashMap<String, Object> ret = new HashMap<String, Object>(); + String serviceName = getServiceName(); + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceKMS.validateConfig Service: (" + serviceName + " )"); + } + if ( configs != null) { + try { + ret = KMSResourceMgr.validateConfig(serviceName, configs); + } catch (Exception e) { + LOG.error("<== RangerServiceKMS.validateConfig Error:" + e); + throw e; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceKMS.validateConfig Response : (" + ret + " )"); + } + return ret; + } + + @Override + public List<String> lookupResource(ResourceLookupContext context) throws Exception { + + List<String> ret = new ArrayList<String>(); + String serviceName = getServiceName(); + Map<String,String> configs = getConfigs(); + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceKMS.lookupResource Context: (" + context + ")"); + } + if (context != null) { + try { + ret = KMSResourceMgr.getKMSResources(serviceName,configs,context); + } catch (Exception e) { + LOG.error( "<==RangerServiceKMS.lookupResource Error : " + e); + throw e; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceKMS.lookupResource Response: (" + ret + ")"); + } + return ret; + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java ---------------------------------------------------------------------- diff --git a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java new file mode 100755 index 0000000..c956914 --- /dev/null +++ b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.kms.client; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.client.BaseClient; +import org.apache.ranger.plugin.client.HadoopException; +import org.apache.ranger.services.kms.client.KMSClient; +import org.apache.ranger.services.kms.client.json.model.KMSSchedulerResponse; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; + +public class KMSClient { + + public static final Logger LOG = Logger.getLogger(KMSClient.class) ; + + private static final String EXPECTED_MIME_TYPE = "application/json"; + + private static final String KMS_LIST_API_ENDPOINT = "/ws/v1/cluster/scheduler" ; + + private static final String errMessage = " You can still save the repository and start creating " + + "policies, but you would not be able to use autocomplete for " + + "resource names. Check xa_portal.log for more info."; + + + String kmsQUrl; + String userName; + String password; + + public KMSClient(String kmsQueueUrl, String kmsUserName, String kmsPassWord) { + + this.kmsQUrl = kmsQueueUrl; + this.userName = kmsUserName ; + this.password = kmsPassWord; + + if (LOG.isDebugEnabled()) { + LOG.debug("Kms Client is build with url [" + kmsQueueUrl + "] user: [" + kmsPassWord + "], password: [" + "" + "]"); + } + + } + + public List<String> getQueueList(final String queueNameMatching, final List<String> existingQueueList) { + + if (LOG.isDebugEnabled()) { + LOG.debug("Getting Kms queue list for queueNameMatching : " + queueNameMatching); + } + final String errMsg = errMessage; + + List<String> ret = new ArrayList<String>(); + + Callable<List<String>> kmsQueueListGetter = new Callable<List<String>>() { + @Override + public List<String> call() { + + List<String> lret = new ArrayList<String>(); + + String url = kmsQUrl + KMS_LIST_API_ENDPOINT ; + + Client client = null ; + ClientResponse response = null ; + + try { + client = Client.create() ; + + WebResource webResource = client.resource(url); + + response = webResource.accept(EXPECTED_MIME_TYPE) + .get(ClientResponse.class); + + if (LOG.isDebugEnabled()) { + LOG.debug("getQueueList():calling " + url); + } + + if (response != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("getQueueList():response.getStatus()= " + response.getStatus()); + } + if (response.getStatus() == 200) { + String jsonString = response.getEntity(String.class); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + KMSSchedulerResponse kmsQResponse = gson.fromJson(jsonString, KMSSchedulerResponse.class); + if (kmsQResponse != null) { + List<String> kmsQueueList = kmsQResponse.getQueueNames(); + if (kmsQueueList != null) { + for ( String kmsQueueName : kmsQueueList) { + if ( existingQueueList != null && existingQueueList.contains(kmsQueueName)) { + continue; + } + if (queueNameMatching == null || queueNameMatching.isEmpty() + || kmsQueueName.startsWith(queueNameMatching)) { + if (LOG.isDebugEnabled()) { + LOG.debug("getQueueList():Adding kmsQueue " + kmsQueueName); + } + lret.add(kmsQueueName) ; + } + } + } + } + } else{ + LOG.info("getQueueList():response.getStatus()= " + response.getStatus() + " for URL " + url + ", so returning null list"); + String jsonString = response.getEntity(String.class); + LOG.info(jsonString); + lret = null; + } + } else { + String msgDesc = "Unable to get a valid response for " + + "expected mime type : [" + EXPECTED_MIME_TYPE + + "] URL : " + url + " - got null response."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, + msgDesc + errMsg, null, null); + throw hdpException; + } + } catch (HadoopException he) { + throw he; + } catch (Throwable t) { + String msgDesc = "Exception while getting Kms Queue List." + + " URL : " + url; + HadoopException hdpException = new HadoopException(msgDesc, + t); + LOG.error(msgDesc, t); + + hdpException.generateResponseDataMap(false, + BaseClient.getMessage(t), msgDesc + errMsg, null, + null); + throw hdpException; + + } finally { + if (response != null) { + response.close(); + } + + if (client != null) { + client.destroy(); + } + + } + return lret ; + } + } ; + + try { + ret = timedTask(kmsQueueListGetter, 5, TimeUnit.SECONDS); + } catch ( Exception e) { + LOG.error("Unable to get Kms Queue list from [" + kmsQUrl + "]", e) ; + } + + return ret; + } + + public static HashMap<String, Object> testConnection(String serviceName, + Map<String, String> configs) { + + List<String> strList = new ArrayList<String>(); + String errMsg = errMessage; + boolean connectivityStatus = false; + HashMap<String, Object> responseData = new HashMap<String, Object>(); + + KMSClient kmsClient = getKmsClient(serviceName, + configs); + strList = getKmsResource(kmsClient, "",null); + + if (strList != null) { + connectivityStatus = true; + } + + if (connectivityStatus) { + String successMsg = "TestConnection Successful"; + BaseClient.generateResponseDataMap(connectivityStatus, successMsg, + successMsg, null, null, responseData); + } else { + String failureMsg = "Unable to retrieve any Kms Queues using given parameters."; + BaseClient.generateResponseDataMap(connectivityStatus, failureMsg, + failureMsg + errMsg, null, null, responseData); + } + + return responseData; + } + + public static KMSClient getKmsClient(String serviceName, + Map<String, String> configs) { + KMSClient kmsClient = null; + if (LOG.isDebugEnabled()) { + LOG.debug("Getting KmsClient for datasource: " + serviceName + + "configMap: " + configs); + } + String errMsg = errMessage; + if (configs == null || configs.isEmpty()) { + String msgDesc = "Could not connect as Connection ConfigMap is empty."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, msgDesc + + errMsg, null, null); + throw hdpException; + } else { + String kmsUrl = configs.get("provider"); + String kmsUserName = configs.get("username"); + String kmsPassWord = configs.get("password"); + kmsClient = new KMSClient (kmsUrl, kmsUserName, + kmsPassWord); + + } + return kmsClient; + } + + public static List<String> getKmsResource (final KMSClient kmsClient, + String yanrQname, List<String> existingQueueName) { + + List<String> resultList = new ArrayList<String>(); + String errMsg = errMessage; + + try { + if (kmsClient == null) { + String msgDesc = "Unable to get Kms Queue : KmsClient is null."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, msgDesc + + errMsg, null, null); + throw hdpException; + } + + if (yanrQname != null) { + String finalkmsQueueName = (yanrQname == null) ? "" + : yanrQname.trim(); + resultList = kmsClient + .getQueueList(finalkmsQueueName,existingQueueName); + if (resultList != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Returning list of " + resultList.size() + " Kms Queues"); + } + } + } + } catch (HadoopException he) { + throw he; + } catch (Exception e) { + String msgDesc = "getKmsResource: Unable to get Kms resources."; + LOG.error(msgDesc, e); + HadoopException hdpException = new HadoopException(msgDesc); + + hdpException.generateResponseDataMap(false, + BaseClient.getMessage(e), msgDesc + errMsg, null, null); + throw hdpException; + } + return resultList; + } + + public static <T> T timedTask(Callable<T> callableObj, long timeout, + TimeUnit timeUnit) throws Exception { + return callableObj.call(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSConnectionMgr.java ---------------------------------------------------------------------- diff --git a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSConnectionMgr.java b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSConnectionMgr.java new file mode 100755 index 0000000..94eaba4 --- /dev/null +++ b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSConnectionMgr.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.kms.client; + +import org.apache.log4j.Logger; + + +public class KMSConnectionMgr { + + public static final Logger LOG = Logger.getLogger(KMSConnectionMgr.class); + + public static KMSClient getKMSClient(final String kmsURL, String userName, String password) { + KMSClient kmsClient = null; + if (kmsURL == null || kmsURL.isEmpty()) { + LOG.error("Can not create KMSClient: kmsURL is empty"); + } else if (userName == null || userName.isEmpty()) { + LOG.error("Can not create KMSClient: kmsuserName is empty"); + } else if (password == null || password.isEmpty()) { + LOG.error("Can not create KMSClient: kmsPassWord is empty"); + } else { + kmsClient = new KMSClient(kmsURL, userName, password); + } + return kmsClient; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java ---------------------------------------------------------------------- diff --git a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java new file mode 100755 index 0000000..36a4840 --- /dev/null +++ b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.kms.client; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.service.ResourceLookupContext; + +public class KMSResourceMgr { + public static final Logger LOG = Logger.getLogger(KMSResourceMgr.class); + private static final String KMSQUEUE = "queue"; + + public static HashMap<String, Object> validateConfig(String serviceName, Map<String, String> configs) throws Exception { + HashMap<String, Object> ret = null; + + if(LOG.isDebugEnabled()) { + LOG.debug("==> KMSResourceMgr.validateConfig ServiceName: "+ serviceName + "Configs" + configs ) ; + } + + try { + ret = KMSClient.testConnection(serviceName, configs); + } catch (Exception e) { + LOG.error("<== KMSResourceMgr.validateConfig Error: " + e) ; + throw e; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== KMSResourceMgr.validateConfig Result : "+ ret ) ; + } + return ret; + } + + public static List<String> getKMSResources(String serviceName, Map<String, String> configs,ResourceLookupContext context) { + String userInput = context.getUserInput(); + Map<String, List<String>> resourceMap = context.getResources(); + List<String> resultList = null; + List<String> kmsQueueList = null; + String kmsQueueName = null; + + if ( resourceMap != null && !resourceMap.isEmpty() && + resourceMap.get(KMSQUEUE) != null ) { + kmsQueueName = userInput; + kmsQueueList = resourceMap.get(KMSQUEUE); + } else { + kmsQueueName = userInput; + } + + + if (configs == null || configs.isEmpty()) { + LOG.error("Connection Config is empty"); + + } else { + + String url = configs.get("kms.url"); + String username = configs.get("username"); + String password = configs.get("password"); + resultList = getKMSResource(url, username, password,kmsQueueName,kmsQueueList) ; + } + return resultList ; + } + + public static List<String> getKMSResource(String url, String username, String password,String kmsQueueName, List<String> kmsQueueList) { + final KMSClient KMSClient = KMSConnectionMgr.getKMSClient(url, username, password); + List<String> topologyList = KMSClient.getQueueList(kmsQueueName, kmsQueueList); + return topologyList; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/75783f72/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/json/model/KMSSchedulerResponse.java ---------------------------------------------------------------------- diff --git a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/json/model/KMSSchedulerResponse.java b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/json/model/KMSSchedulerResponse.java new file mode 100755 index 0000000..1e696be --- /dev/null +++ b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/json/model/KMSSchedulerResponse.java @@ -0,0 +1,105 @@ +package org.apache.ranger.services.kms.client.json.model; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class KMSSchedulerResponse implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private KMSScheduler scheduler = null; + + public KMSScheduler getScheduler() { return scheduler; } + + public List<String> getQueueNames() { + List<String> ret = new ArrayList<String>(); + + if(scheduler != null) { + scheduler.collectQueueNames(ret); + } + + return ret; + } + + + @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class KMSScheduler implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private KMSSchedulerInfo schedulerInfo = null; + + public KMSSchedulerInfo getSchedulerInfo() { return schedulerInfo; } + + public void collectQueueNames(List<String> queueNames) { + if(schedulerInfo != null) { + schedulerInfo.collectQueueNames(queueNames, null); + } + } + } + + @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class KMSSchedulerInfo implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String queueName = null; + private KMSQueues queues = null; + + public String getQueueName() { return queueName; } + + public KMSQueues getQueues() { return queues; } + + public void collectQueueNames(List<String> queueNames, String parentQueueName) { + if(queueName != null) { + String queueFqdn = parentQueueName == null ? queueName : parentQueueName + "." + queueName; + + queueNames.add(queueFqdn); + + if(queues != null) { + queues.collectQueueNames(queueNames, queueFqdn); + } + } + } + } + + @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class KMSQueues implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private List<KMSSchedulerInfo> queue = null; + + public List<KMSSchedulerInfo> getQueue() { return queue; } + + public void collectQueueNames(List<String> queueNames, String parentQueueName) { + if(queue != null) { + for(KMSSchedulerInfo schedulerInfo : queue) { + schedulerInfo.collectQueueNames(queueNames, parentQueueName); + } + } + } + } +}
