Repository: incubator-ranger Updated Branches: refs/heads/master 6ac22dbb9 -> aceff0e8e
RANGER-248: Ranger plugin for YARN authorization Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/c0bb7ecc Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/c0bb7ecc Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/c0bb7ecc Branch: refs/heads/master Commit: c0bb7ecc6d8e0e28fe645314f87428cbccedbc70 Parents: 5580ca2 Author: Madhan Neethiraj <[email protected]> Authored: Mon Feb 23 01:59:19 2015 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Feb 23 01:59:19 2015 -0800 ---------------------------------------------------------------------- .../plugin/store/EmbeddedServiceDefsUtil.java | 8 +- .../ranger/plugin/util/GrantRevokeRequest.java | 2 +- .../service-defs/ranger-servicedef-yarn.json | 35 ++ plugin-yarn/.gitignore | 1 + .../conf/ranger-policymgr-ssl-changes.cfg | 23 ++ plugin-yarn/conf/ranger-policymgr-ssl.xml | 63 ++++ plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 33 ++ plugin-yarn/conf/ranger-yarn-audit.xml | 187 ++++++++++ .../conf/ranger-yarn-security-changes.cfg | 26 ++ plugin-yarn/conf/ranger-yarn-security.xml | 67 ++++ plugin-yarn/conf/yarn-site-changes.cfg | 17 + plugin-yarn/pom.xml | 61 ++++ plugin-yarn/scripts/install.properties | 106 ++++++ plugin-yarn/scripts/install.sh | 354 +++++++++++++++++++ plugin-yarn/scripts/uninstall.sh | 65 ++++ .../scripts/yarn-plugin-install.properties | 23 ++ .../yarn/authorizer/RangerYarnAuthorizer.java | 310 ++++++++++++++++ pom.xml | 5 +- .../org/apache/ranger/biz/ServiceDBStore.java | 138 ++++---- .../org/apache/ranger/rest/ServiceREST.java | 52 ++- src/main/assembly/plugin-yarn.xml | 151 ++++++++ 21 files changed, 1644 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index a7ad7b1..75cf905 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -48,6 +48,7 @@ public class EmbeddedServiceDefsUtil { public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; public static final String EMBEDDED_SERVICEDEF_KNOX_NAME = "knox"; public static final String EMBEDDED_SERVICEDEF_STORM_NAME = "storm"; + public static final String EMBEDDED_SERVICEDEF_YARN_NAME = "yarn"; public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; private static EmbeddedServiceDefsUtil instance = new EmbeddedServiceDefsUtil(); @@ -58,6 +59,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef hiveServiceDef = null; private RangerServiceDef knoxServiceDef = null; private RangerServiceDef stormServiceDef = null; + private RangerServiceDef yarnServiceDef = null; private Gson gsonBuilder = null; @@ -83,11 +85,11 @@ public class EmbeddedServiceDefsUtil { hiveServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HIVE_NAME); knoxServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KNOX_NAME); stormServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_STORM_NAME); + yarnServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_YARN_NAME); } catch(Throwable excp) { LOG.fatal("EmbeddedServiceDefsUtil.init(): failed", excp); } - LOG.info("<== EmbeddedServiceDefsUtil.init()"); } @@ -111,6 +113,10 @@ public class EmbeddedServiceDefsUtil { return getId(stormServiceDef); } + public long getYarnServiceDefId() { + return getId(yarnServiceDef); + } + private long getId(RangerServiceDef serviceDef) { return serviceDef == null || serviceDef.getId() == null ? -1 : serviceDef.getId().longValue(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/agents-common/src/main/java/org/apache/ranger/plugin/util/GrantRevokeRequest.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/GrantRevokeRequest.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/GrantRevokeRequest.java index b40ea18..cf871ca 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/GrantRevokeRequest.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/GrantRevokeRequest.java @@ -136,7 +136,7 @@ public class GrantRevokeRequest implements Serializable { * @param accessTypes the accessTypes to set */ public void setAccessTypes(Set<String> accessTypes) { - this.accessTypes = accessTypes == null ? new HashSet<String>() : groups; + this.accessTypes = accessTypes == null ? new HashSet<String>() : accessTypes; } /** http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json new file mode 100644 index 0000000..00674b1 --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json @@ -0,0 +1,35 @@ +{ + "id":7, + "name":"yarn", + "implClass":"org.apache.ranger.services.yarn.RangerServiceYarn", + "label":"YARN", + "description":"YARN", + "guid":"5b710438-edcf-4e20-834c-a9a267b5b963", + "createTime":"20141208-22:55:47.095--0800", + "updateTime":"20141208-22:55:47.095--0800", + "version":1, + "enums": + [ + ], + "configs": + [ + {"name":"username", "type":"string", "mandatory":true, "label":"Username"}, + {"name":"password", "type":"password","mandatory":true, "label":"Password"}, + {"name":"yarn.url", "type":"string", "mandatory":true, "defaultValue":"","label":"YARN REST URL"}, + {"name":"certificate.cn","type":"string", "mandatory":false,"label":"Common Name for Certificate"} + ], + "resources": + [ + {"name":"queue","type":"string","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher","matcherOptions":"wildCard=true;ignoreCase=true;pathSeperatorChar=.","label":"Queue","description":"Queue"} + ], + "accessTypes": + [ + {"name":"submit-app","label":"submit-app"}, + {"name":"admin-queue","label":"admin-queue"}, + {"name":"admin","label":"admin"} + ], + "policyConditions": + [ + {"name":"ip-range","evaluator":"org.apache.ranger.knox.IpRangeCondition","evaluatorOptions":"","label":"IP Address Range","description":"IP Address Range"} + ] +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/.gitignore ---------------------------------------------------------------------- diff --git a/plugin-yarn/.gitignore b/plugin-yarn/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/plugin-yarn/.gitignore @@ -0,0 +1 @@ +/target http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/conf/ranger-policymgr-ssl-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-policymgr-ssl-changes.cfg b/plugin-yarn/conf/ranger-policymgr-ssl-changes.cfg new file mode 100644 index 0000000..ec4eeab --- /dev/null +++ b/plugin-yarn/conf/ranger-policymgr-ssl-changes.cfg @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SSL Params +# +xasecure.policymgr.clientssl.keystore %SSL_KEYSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.keystore.password %SSL_KEYSTORE_PASSWORD% mod create-if-not-exists +xasecure.policymgr.clientssl.keystore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore %SSL_TRUSTSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore.password %SSL_TRUSTSTORE_PASSWORD% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/conf/ranger-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-policymgr-ssl.xml b/plugin-yarn/conf/ranger-policymgr-ssl.xml new file mode 100644 index 0000000..964aac7 --- /dev/null +++ b/plugin-yarn/conf/ranger-policymgr-ssl.xml @@ -0,0 +1,63 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <!-- The following properties are used for 2-way SSL client server validation --> + <property> + <name>xasecure.policymgr.clientssl.keystore</name> + <value>hadoopdev-clientcert.jks</value> + <description> + Java Keystore files + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.password</name> + <value>none</value> + <description> + password for keystore + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore</name> + <value>cacerts-xasecure.jks</value> + <description> + java truststore file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.password</name> + <value>none</value> + <description> + java truststore password + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.credential.file</name> + <value>jceks://file/tmp/keystore-hadoopdev-ssl.jceks</value> + <description> + java keystore credential file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.credential.file</name> + <value>jceks://file/tmp/truststore-hadoopdev-ssl.jceks</value> + <description> + java truststore credential file + </description> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/conf/ranger-yarn-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg new file mode 100644 index 0000000..4f2c5a2 --- /dev/null +++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT_DB_JDBC_URL% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists +xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT_DB_JDBC_DRIVER% mod create-if-not-exists + +xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.directory %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.directory %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/conf/ranger-yarn-audit.xml ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml new file mode 100644 index 0000000..c0096a4 --- /dev/null +++ b/plugin-yarn/conf/ranger-yarn-audit.xml @@ -0,0 +1,187 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>xasecure.audit.is.enabled</name> + <value>true</value> + </property> + + + <!-- DB audit provider configuration --> + <property> + <name>xasecure.audit.db.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.db.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.db.batch.size</name> + <value>100</value> + </property> + + <!-- Properties whose name begin with "xasecure.audit.jpa." are used to configure JPA --> + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.url</name> + <value>jdbc:mysql://localhost:3306/ranger_audit</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.user</name> + <value>rangerlogger</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.password</name> + <value>none</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.driver</name> + <value>com.mysql.jdbc.Driver</value> + </property> + + <property> + <name>xasecure.audit.credential.provider.file</name> + <value>jceks://file/etc/ranger/yarndev/auditcred.jceks</value> + </property> + + + + <!-- HDFS audit provider configuration --> + <property> + <name>xasecure.audit.hdfs.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.hdfs.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.queue.size</name> + <value>1048576</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.encoding</name> + <value></value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.directory</name> + <value>hdfs://NAMENODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.file</name> + <value>%hostname%-audit.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name> + <value>900</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name> + <value>86400</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.directory</name> + <value>/var/log/yarn/audit</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file</name> + <value>%time:yyyyMMdd-HHmm.ss%.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file.buffer.size.bytes</name> + <value>8192</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name> + <value>600</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.directory</name> + <value>/var/log/yarn/audit/archive</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name> + <value>10</value> + </property> + + + <!-- Log4j audit provider configuration --> + <property> + <name>xasecure.audit.log4j.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.is.async</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> + <value>30000</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/conf/ranger-yarn-security-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-security-changes.cfg b/plugin-yarn/conf/ranger-yarn-security-changes.cfg new file mode 100644 index 0000000..87fa972 --- /dev/null +++ b/plugin-yarn/conf/ranger-yarn-security-changes.cfg @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Change the original policy parameter to work with policy manager based. +# +# +ranger.plugin.yarn.service.name %REPOSITORY_NAME% mod create-if-not-exists + +ranger.plugin.yarn.policy.source.impl org.apache.ranger.admin.client.RangerAdminRESTClient mod create-if-not-exists + +ranger.plugin.yarn.policy.rest.url %POLICY_MGR_URL% mod create-if-not-exists +ranger.plugin.yarn.policy.rest.ssl.config.file /etc/hadoop/conf/ranger-policymgr-ssl.xml mod create-if-not-exists +ranger.plugin.yarn.policy.pollIntervalMs 30000 mod create-if-not-exists +ranger.plugin.yarn.policy.cache.dir %POLICY_CACHE_FILE_PATH% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/conf/ranger-yarn-security.xml ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-security.xml b/plugin-yarn/conf/ranger-yarn-security.xml new file mode 100644 index 0000000..f6e37f8 --- /dev/null +++ b/plugin-yarn/conf/ranger-yarn-security.xml @@ -0,0 +1,67 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>ranger.plugin.yarn.service.name</name> + <value>yarndev</value> + <description> + Name of the Ranger service containing policies for this YARN instance + </description> + </property> + + <property> + <name>ranger.plugin.yarn.policy.source.impl</name> + <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value> + <description> + Class to retrieve policies from the source + </description> + </property> + + <property> + <name>ranger.plugin.yarn.policy.rest.url</name> + <value>http://policymanagerhost:port</value> + <description> + URL to Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.yarn.policy.rest.ssl.config.file</name> + <value>/etc/hadoop/conf/ranger-policymgr-ssl.xml</value> + <description> + Path to the file containing SSL details to contact Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.yarn.policy.pollIntervalMs</name> + <value>30000</value> + <description> + How often to poll for changes in policies? + </description> + </property> + + <property> + <name>ranger.plugin.yarn.policy.cache.dir</name> + <value>/etc/ranger/hadoopdev/policycache</value> + <description> + Directory where Ranger policies are cached after successful retrieval from the source + </description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/conf/yarn-site-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/yarn-site-changes.cfg b/plugin-yarn/conf/yarn-site-changes.cfg new file mode 100644 index 0000000..ed1d471 --- /dev/null +++ b/plugin-yarn/conf/yarn-site-changes.cfg @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +yarn.acl.enable true mod create-if-not-exists +yarn.authorization-provider org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/pom.xml ---------------------------------------------------------------------- diff --git a/plugin-yarn/pom.xml b/plugin-yarn/pom.xml new file mode 100644 index 0000000..1bb3b7a --- /dev/null +++ b/plugin-yarn/pom.xml @@ -0,0 +1,61 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>security_plugins.ranger-yarn-plugin</groupId> + <artifactId>ranger-yarn-plugin</artifactId> + <name>YARN Security Plugin</name> + <description>YARN Security Plugin</description> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <parent> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger</artifactId> + <version>0.4.0</version> + <relativePath>..</relativePath> + </parent> + <dependencies> + <dependency> + <groupId>security_plugins.ranger-plugins-common</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>security_plugins.ranger-plugins-audit</groupId> + <artifactId>ranger-plugins-audit</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>credentialbuilder</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <version>${hadoop.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/scripts/install.properties ---------------------------------------------------------------------- diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties new file mode 100644 index 0000000..d2d1ffe --- /dev/null +++ b/plugin-yarn/scripts/install.properties @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=yarndev +# +REPOSITORY_NAME= + +# +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=false +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME= +XAAUDIT.DB.DATABASE_NAME= +XAAUDIT.DB.USER_NAME= +XAAUDIT.DB.PASSWORD= + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/yarn/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/yarn/audit/archive +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/yarn/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/yarn/audit/archive +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/yarn/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/yarn/audit/archive + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/scripts/install.sh ---------------------------------------------------------------------- diff --git a/plugin-yarn/scripts/install.sh b/plugin-yarn/scripts/install.sh new file mode 100644 index 0000000..859b80c --- /dev/null +++ b/plugin-yarn/scripts/install.sh @@ -0,0 +1,354 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +create_jceks() +{ + alias=$1 + pass=$2 + jceksFile=$3 + + java -cp "${install_dir}/cred/lib/*:${install_dir}/installer/lib/*" org.apache.ranger.credentialapi.buildks create ${alias} -value ${pass} -provider jceks://file${jceksFile} + if [ $? -ne 0 ] + then + echo "ERROR: Unable to create/update credential file [${jceksFile}] for alias [${alias}]" + exit 1 + fi +} + +#Update Properties to File +#$1 -> propertyName $2 -> newPropertyValue $3 -> fileName +updatePropertyToFile(){ + sed -i 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 + #validate=`sed -i 's/^'$1'=[^ ]*$/'$1'='$2'/g' $3` #for validation + validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation + #echo 'V1:'$validate + if test -z "$validate" ; then echo "[E] '$1' not found in $3 file while Updating....!!"; exit 1; fi + echo "[I] File $3 Updated successfully : {'$1'}" +} + +yarn_dir=/usr/lib/yarn +yarn_lib_dir=${yarn_dir}/lib +yarn_conf_dir=/etc/yarn/conf +yarn_bin_dir=${yarn_dir}/bin + +CONFIG_FILE_OWNER=yarn:yarn + +yarn_srv_conf_dir=${yarn_conf_dir} +yarn_cli_conf_dir="${yarn_conf_dir}" + +install_dir=`dirname $0` + +[ "${install_dir}" = "." ] && install_dir=`pwd` + + +#verify sql-connector path is valid +SQL_CONNECTOR_JAR=`grep '^SQL_CONNECTOR_JAR' ${install_dir}/install.properties | awk -F= '{ print $2 }'` +echo "[I] Checking SQL CONNECTOR FILE : $SQL_CONNECTOR_JAR" +if test -f "$SQL_CONNECTOR_JAR"; then + echo "[I] SQL CONNECTOR FILE : $SQL_CONNECTOR_JAR file found" +else + echo "[E] SQL CONNECTOR FILE : $SQL_CONNECTOR_JAR not found, aborting installation" + exit 1 +fi +#copying sql connector jar file to lib directory +cp $SQL_CONNECTOR_JAR ${install_dir}/lib + +#echo "Current Install Directory: [${install_dir}]" + + +# +# --- Backup current configuration for backup - START +# + +COMPONENT_NAME=yarn + +XASECURE_VERSION=`cat ${install_dir}/version` + +CFG_DIR=${yarn_conf_dir} +XASECURE_ROOT=/etc/xasecure/${COMPONENT_NAME} +BACKUP_TYPE=pre +CUR_VERSION_FILE=${XASECURE_ROOT}/.current_version +CUR_CFG_DIR_FILE=${XASECURE_ROOT}/.config_dir +PRE_INSTALL_CONFIG=${XASECURE_ROOT}/${BACKUP_TYPE}-${XASECURE_VERSION} + +if [ ! -d ${XASECURE_ROOT} ] +then + mkdir -p ${XASECURE_ROOT} +fi + +backup_dt=`date '+%Y%m%d%H%M%S'` + +if [ -d "${PRE_INSTALL_CONFIG}" ] +then + PRE_INSTALL_CONFIG="${PRE_INSTALL_CONFIG}.${backup_dt}" +fi + +if [ -d ${CFG_DIR} ] +then + ( cd ${CFG_DIR} ; find . -print | cpio -pdm ${PRE_INSTALL_CONFIG} ) + [ -f ${CUR_VERSION_FILE} ] && mv ${CUR_VERSION_FILE} ${CUR_VERSION_FILE}-${backup_dt} + echo ${XASECURE_VERSION} > ${CUR_VERSION_FILE} + echo ${CFG_DIR} > ${CUR_CFG_DIR_FILE} +else + echo "+ mkdir -p ${CFG_DIR} ..." + mkdir -p ${CFG_DIR} +fi + +cp -f ${install_dir}/uninstall.sh ${XASECURE_ROOT}/ + +# +# --- Backup current configuration for backup - END +# + + +dt=`date '+%Y%m%d%H%M%S'` +for f in ${install_dir}/conf/* +do + if [ -f ${f} ] + then + fn=`basename $f` + if [ ! -f ${yarn_conf_dir}/${fn} ] + then + echo "+cp ${f} ${yarn_conf_dir}/${fn}" + cp ${f} ${yarn_conf_dir}/${fn} + else + echo "WARN: ${fn} already exists in the ${yarn_conf_dir} - Using existing configuration ${fn}" + fi + fi +done + + +if [ ! -d ${yarn_lib_dir} ] +then + echo "+mkdir -p ${yarn_lib_dir}" + mkdir -p ${yarn_lib_dir} +fi + +for f in ${install_dir}/dist/*.jar ${install_dir}/lib/*.jar +do + if [ -f ${f} ] + then + fn=`basename $f` + echo "+cp ${f} ${yarn_lib_dir}/${fn}" + cp ${f} ${yarn_lib_dir}/${fn} + fi +done + +# +# Copy the SSL parameters +# + +CredFile=`grep '^CREDENTIAL_PROVIDER_FILE' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + +if ! [ `echo ${CredFile} | grep '^/.*'` ] +then + echo "ERROR:Please enter the Credential File Store with proper file path" + exit 1 +fi +pardir=`dirname ${CredFile}` + +if [ ! -d ${pardir} ] +then + mkdir -p ${pardir} + chmod go+rx ${pardir} +fi + +# +# Generate Credential Provider file and Credential for Audit DB access. +# + + +auditCredAlias="auditDBCred" + +auditdbCred=`grep '^XAAUDIT.DB.PASSWORD' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + +create_jceks ${auditCredAlias} ${auditdbCred} ${CredFile} + + +# +# Generate Credential Provider file and Credential for SSL KEYSTORE AND TRUSTSTORE +# + + +sslkeystoreAlias="sslKeyStore" + +sslkeystoreCred=`grep '^SSL_KEYSTORE_PASSWORD' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + +create_jceks ${sslkeystoreAlias} ${sslkeystoreCred} ${CredFile} + + +ssltruststoreAlias="sslTrustStore" + +ssltruststoreCred=`grep '^SSL_TRUSTSTORE_PASSWORD' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + +create_jceks ${ssltruststoreAlias} ${ssltruststoreCred} ${CredFile} + +chown ${CONFIG_FILE_OWNER} ${CredFile} + +PROP_ARGS="-p ${install_dir}/install.properties" + +to_file="${install_dir}/install.properties" +DB_FLAVOR=`grep '^XAAUDIT.DB.FLAVOUR' ${install_dir}/install.properties | awk -F= '{ print $2 }'` +DB_FLAVOR=`echo $DB_FLAVOR | tr '[:lower:]' '[:upper:]'` +if [ "${DB_FLAVOR}" == "" ] +then + $DB_FLAVOR="MYSQL" +fi +echo "[I] Updating install.properites setting for : $DB_FLAVOR" + +if [ "${DB_FLAVOR}" == "MYSQL" ] +then + audit_db_hostname=`grep '^XAAUDIT.DB.HOSTNAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + audit_db_name=`grep '^XAAUDIT.DB.DATABASE_NAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + propertyName=XAAUDIT.DB.JDBC_URL + newPropertyValue="jdbc:mysql://${audit_db_hostname}/${audit_db_name}" + updatePropertyToFile $propertyName $newPropertyValue $to_file + + propertyName=XAAUDIT.DB.JDBC_DRIVER + newPropertyValue="com.mysql.jdbc.Driver" + updatePropertyToFile $propertyName $newPropertyValue $to_file +fi + +if [ "${DB_FLAVOR}" == "ORACLE" ] +then + audit_db_hostname=`grep '^XAAUDIT.DB.HOSTNAME' ${install_dir}/install.properties | awk -F= '{ print $2 }'` + propertyName=XAAUDIT.DB.JDBC_URL + newPropertyValue="jdbc:oracle:thin:\@//${audit_db_hostname}" + updatePropertyToFile $propertyName $newPropertyValue $to_file + + propertyName=XAAUDIT.DB.JDBC_DRIVER + newPropertyValue="oracle.jdbc.OracleDriver" + updatePropertyToFile $propertyName $newPropertyValue $to_file +fi +for f in ${install_dir}/installer/conf/*-changes.cfg +do + if [ -f ${f} ] + then + fn=`basename $f` + orgfn=`echo $fn | sed -e 's:-changes.cfg:.xml:'` + fullpathorgfn="${yarn_conf_dir}/${orgfn}" + if [ ! -f ${fullpathorgfn} ] + then + echo "ERROR: Unable to find ${fullpathorgfn}" + exit 1 + fi + archivefn="${yarn_conf_dir}/.${orgfn}.${dt}" + newfn="${yarn_conf_dir}/.${orgfn}-new.${dt}" + cp ${fullpathorgfn} ${archivefn} + if [ $? -eq 0 ] + then + cp="${install_dir}/installer/lib/*:${install_dir}/cred/lib/*:" + java -cp "${cp}" org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} ${PROP_ARGS} + if [ $? -eq 0 ] + then + diff -w ${newfn} ${fullpathorgfn} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + #echo "Changing config file: ${fullpathorgfn} with following changes:" + #echo "===============================================================" + #diff -w ${newfn} ${fullpathorgfn} + #echo "===============================================================" + echo "NOTE: Current config file: ${fullpathorgfn} is being saved as ${archivefn}" + #echo "===============================================================" + cp ${newfn} ${fullpathorgfn} + fi + else + echo "ERROR: Unable to make changes to config. file: ${fullpathorgfn}" + echo "exiting ...." + exit 1 + fi + else + echo "ERROR: Unable to save config. file: ${fullpathorgfn} to ${archivefn}" + echo "exiting ...." + exit 1 + fi + fi +done + +chmod go-rwx ${yarn_conf_dir}/xasecure-policymgr-ssl.xml +chown ${CONFIG_FILE_OWNER} ${yarn_conf_dir}/xasecure-policymgr-ssl.xml + +# +# Adding authorizer to yarn.yaml configuration file ... +# +YARN_DIR=/etc/yarn +YARN_CONFIG_FILE=yarn.yaml +YARN_BIN_FILE=/usr/bin/yarn + +dt=`date '+%Y%m%d%H%M%S'` +CONFIG_FILE=${YARN_DIR}/${YARN_CONFIG_FILE} +ARCHIVE_FILE=${YARN_DIR}/.${YARN_CONFIG_FILE}.${dt} +YARN_BIN_ARCHIVE_FILE=/usr/bin/.yarn.${dt} + +cp ${CONFIG_FILE} ${ARCHIVE_FILE} + +awk -F: 'BEGIN { + configured = 0 ; +} +{ + if ($1 == "nimbus.authorizer") { + if ($2 ~ /^[ \t]*"org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer"[ \t]*$/) { + configured = 1 ; + printf("%s\n",$0) ; + } + else { + printf("#%s\n",$0); + printf("nimbus.authorizer: \"org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer\"\n") ; + configured = 1 ; + } + } + else { + printf("%s\n",$0) ; + } +} +END { + if (configured == 0) { + printf("nimbus.authorizer: \"org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer\"\n") ; + } +}' ${ARCHIVE_FILE} > ${ARCHIVE_FILE}.new + +if [ ! -z ${ARCHIVE_FILE}.new ] +then + cat ${ARCHIVE_FILE}.new > ${CONFIG_FILE} + rm -f ${ARCHIVE_FILE}.new +else + echo "ERROR: ${ARCHIVE_FILE}.new file has not created successfully." + exit 1 +fi + +# +# Modify the CLASSPATH of the YARN Servers (ui) .... +# +grep 'ret.extend(\["/etc/yarn/conf"' ${YARN_BIN_FILE} > /dev/null +if [ $? -ne 0 ] +then + temp=/tmp/yarn.tmp.$$ + cat ${YARN_BIN_FILE} | sed -e '/ret = get_jars_full(YARN_DIR)/ a\ + ret.extend(["/etc/yarn/conf","/usr/lib/yarn/lib/*"])' > ${temp} + if [ ! -z ${temp} ] + then + cp ${YARN_BIN_FILE} ${YARN_BIN_ARCHIVE_FILE} + cat ${temp} > ${YARN_BIN_FILE} + else + echo "ERROR: ${temp} file has not been created successfully." + exit 1 + fi +fi + + +exit 0 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/scripts/uninstall.sh ---------------------------------------------------------------------- diff --git a/plugin-yarn/scripts/uninstall.sh b/plugin-yarn/scripts/uninstall.sh new file mode 100644 index 0000000..d4661e7 --- /dev/null +++ b/plugin-yarn/scripts/uninstall.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Replacing authorizer to yarn.yaml configuration file ... +# +YARN_DIR=/etc/yarn +YARN_CONFIG_FILE=yarn.yaml + +dt=`date '+%Y%m%d%H%M%S'` +CONFIG_FILE=${YARN_DIR}/${YARN_CONFIG_FILE} +ARCHIVE_FILE=${YARN_DIR}/.${YARN_CONFIG_FILE}.${dt} + +cp ${CONFIG_FILE} ${ARCHIVE_FILE} + +awk -F: 'BEGIN { + configured = 0 ; +} +{ + if ($1 == "nimbus.authorizer") { + if ($2 ~ /^[ \t]*"backtype.yarn.security.auth.authorizer.SimpleACLAuthorizer"[ \t]*$/) { + configured = 1 ; + printf("%s\n",$0) ; + } + else { + printf("#%s\n",$0); + printf("nimbus.authorizer: \"backtype.yarn.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; + configured = 1 ; + } + } + else { + printf("%s\n",$0) ; + } +} +END { + if (configured == 0) { + printf("nimbus.authorizer: \"backtype.yarn.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; + } +}' ${ARCHIVE_FILE} > ${ARCHIVE_FILE}.new + +if [ ! -z ${ARCHIVE_FILE}.new ] +then + cat ${ARCHIVE_FILE}.new > ${CONFIG_FILE} + rm -f ${ARCHIVE_FILE}.new + echo "Apache Ranger Plugin has been uninstalled from YARN Service. Please restart YARN nimbus and ui services ..." +else + echo "ERROR: ${ARCHIVE_FILE}.new file has not created successfully." + exit 1 +fi + +exit 0 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/scripts/yarn-plugin-install.properties ---------------------------------------------------------------------- diff --git a/plugin-yarn/scripts/yarn-plugin-install.properties b/plugin-yarn/scripts/yarn-plugin-install.properties new file mode 100644 index 0000000..074bc4d --- /dev/null +++ b/plugin-yarn/scripts/yarn-plugin-install.properties @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# installation properties for this plugin + +# +# Name of the directory where the component's lib and conf directory exist. +# This location should be relative to the parent of the directory containing +# the plugin installation files. +# +COMPONENT_INSTALL_DIR_NAME=hadoop http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java ---------------------------------------------------------------------- diff --git a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java new file mode 100644 index 0000000..e911fcc --- /dev/null +++ b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java @@ -0,0 +1,310 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.authorization.yarn.authorizer; + +import java.net.InetAddress; +import java.security.Permissions; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections.SetUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.security.*; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.policyengine.RangerResource; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.GrantRevokeRequest; + +import com.google.common.collect.Sets; + +public class RangerYarnAuthorizer extends YarnAuthorizationProvider { + public static final String ACCESS_TYPE_ADMIN_QUEUE = "admin-queue"; + public static final String ACCESS_TYPE_SUBMIT_APP = "submit-app"; + public static final String ACCESS_TYPE_ADMIN = "admin"; + + private static final Log LOG = LogFactory.getLog(RangerYarnAuthorizer.class); + + private static volatile RangerYarnPlugin yarnPlugin = null; + + private AccessControlList admins = null; + + @Override + public void init(Configuration conf) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuthorizer.init()"); + } + + RangerYarnPlugin plugin = yarnPlugin; + + if(plugin == null) { + synchronized(RangerYarnAuthorizer.class) { + plugin = yarnPlugin; + + if(plugin == null) { + plugin = new RangerYarnPlugin(); + plugin.init(); + + yarnPlugin = plugin; + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuthorizer.init()"); + } + } + + @Override + public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, UserGroupInformation ugi) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuthorizer.checkPermission(" + accessType + ", " + entity + ", " + ugi + ")"); + } + + boolean ret = false; + + RangerYarnPlugin plugin = yarnPlugin; + + if(plugin != null) { + RangerYarnAccessRequest request = new RangerYarnAccessRequest(entity, getRangerAccessType(accessType), accessType.name(), ugi); + + RangerAccessResult result = plugin.isAccessAllowed(request); + + ret = result == null ? false : result.getIsAllowed(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuthorizer.checkPermission(" + accessType + ", " + entity + ", " + ugi + "): " + ret); + } + + return ret; + } + + @Override + public boolean isAdmin(UserGroupInformation ugi) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuthorizer.isAdmin(" + ugi + ")"); + } + + boolean ret = false; + + if(admins != null) { + ret = admins.isUserAllowed(ugi); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuthorizer.isAdmin(" + ugi + "): " + ret); + } + + return ret; + } + + @Override + public void setAdmins(AccessControlList acl, UserGroupInformation ugi) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuthorizer.setAdmins(" + acl + ", " + ugi + ")"); + } + + admins = acl; + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuthorizer.setAdmins(" + acl + ", " + ugi + ")"); + } + } + + @Override + public void setPermission(PrivilegedEntity entity, Map<AccessType, AccessControlList> permission, UserGroupInformation ugi) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuthorizer.setPermission(" + entity + ", " + permission + ", " + ugi + ")"); + } + + RangerYarnPlugin plugin = yarnPlugin; + + if(plugin != null && entity != null && !MapUtils.isEmpty(permission) && ugi != null) { + RangerYarnResource resource = new RangerYarnResource(entity); + + GrantRevokeRequest request = new GrantRevokeRequest(); + request.setResource(resource.getResourceAsMap()); + request.setGrantor(ugi.getShortUserName()); + request.setDelegateAdmin(Boolean.FALSE); + request.setEnableAudit(Boolean.TRUE); + request.setReplaceExistingPermissions(Boolean.FALSE); + + for(Map.Entry<AccessType, AccessControlList> e : permission.entrySet()) { + AccessType accessType = e.getKey(); + AccessControlList acl = e.getValue(); + + Set<String> accessTypes = new HashSet<String>(); + accessTypes.add(getRangerAccessType(accessType)); + + if(acl.isAllAllowed()) { + Set<String> publicGroup = new HashSet<String>(); + publicGroup.add(RangerPolicyEngine.GROUP_PUBLIC); + + request.setAccessTypes(accessTypes); + request.setUsers(null); + request.setGroups(publicGroup); + } else { + request.setAccessTypes(accessTypes); + request.setUsers(getSet(acl.getUsers())); + request.setGroups(getSet(acl.getGroups())); + } + + LOG.error("==> grantAccess(" + request + ")"); + + try { + plugin.grantAccess(request, plugin.getDefaultAuditHandler()); + } catch(Exception excp) { + LOG.error("grantAccess(" + request + ") failed", excp); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuthorizer.setPermission(" + entity + ", " + permission + ", " + ugi + ")"); + } + } + + private static String getRangerAccessType(AccessType accessType) { + String ret = null; + + switch(accessType) { + case ADMINISTER_QUEUE: + ret = RangerYarnAuthorizer.ACCESS_TYPE_ADMIN_QUEUE; + break; + + case SUBMIT_APP: + ret = RangerYarnAuthorizer.ACCESS_TYPE_SUBMIT_APP; + break; + } + + return ret; + } + + private Set<String> getSet(Collection<String> strings) { + Set<String> ret = null; + + if(! CollectionUtils.isEmpty(strings)) { + if(strings instanceof Set<?>) { + ret = (Set<String>)strings; + } else { + ret = new HashSet<String>(); + for(String str : strings) { + ret.add(str); + } + } + } + + return ret; + } +} + +class RangerYarnPlugin extends RangerBasePlugin { + public RangerYarnPlugin() { + super("yarn", "yarn"); + } + + @Override + public void init() { + super.init(); + + RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); + + super.setDefaultAuditHandler(auditHandler); + } +} + +class RangerYarnResource implements RangerResource { + private static final String KEY_QUEUE = "queue"; + private static final Set<String> KEYS_QUEUE = Sets.newHashSet(KEY_QUEUE); + + private String queue = null; + + public RangerYarnResource(PrivilegedEntity entity) { + this.queue = entity != null ? entity.getName() : null; + } + + @Override + public String getOwnerUser() { + return null; + } + + @Override + public boolean exists(String name) { + return !StringUtils.isEmpty(queue) && StringUtils.equals(name, KEY_QUEUE); + } + + @Override + public String getValue(String name) { + return StringUtils.equals(name, KEY_QUEUE) ? queue : null; + } + + @Override + public Set<String> getKeys() { + return StringUtils.isEmpty(queue) ? Collections.<String>emptySet() : KEYS_QUEUE; + } + + public Map<String, String> getResourceAsMap() { + Map<String, String> ret = new HashMap<String, String>(); + + if(!StringUtils.isEmpty(queue)) { + ret.put(KEY_QUEUE, queue); + } + + return ret; + } +} + +class RangerYarnAccessRequest extends RangerAccessRequestImpl { + public RangerYarnAccessRequest(PrivilegedEntity entity, String accessType, String action, UserGroupInformation ugi) { + super.setResource(new RangerYarnResource(entity)); + super.setAccessType(accessType); + super.setUser(ugi.getShortUserName()); + super.setUserGroups(Sets.newHashSet(ugi.getGroupNames())); + super.setAccessTime(StringUtil.getUTCDate()); + super.setClientIPAddress(getRemoteIp()); + super.setAction(accessType); + } + + private static String getRemoteIp() { + String ret = null ; + InetAddress ip = Server.getRemoteIp() ; + if (ip != null) { + ret = ip.getHostAddress(); + } + return ret ; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 072e7a9..3eec63d 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,6 @@ <modules> <module>jisql</module> - <module>kms</module> <module>agents-audit</module> <module>agents-common</module> <module>agents-cred</module> @@ -87,6 +86,7 @@ <module>hive-agent</module> <module>knox-agent</module> <module>storm-agent</module> + <module>plugin-yarn</module> <module>lookup-client</module> <module>security-admin</module> <module>ugsync</module> @@ -126,7 +126,7 @@ <guava.version>11.0.2</guava.version> <hadoop-auth.version>2.6.0</hadoop-auth.version> <hadoop-common.version>2.6.0</hadoop-common.version> - <hadoop.version>2.6.0</hadoop.version> + <hadoop.version>3.0.0-SNAPSHOT</hadoop.version> <hamcrest.all.version>1.3</hamcrest.all.version> <hbase.version>0.99.2</hbase.version> <hive.version>1.2.0-SNAPSHOT</hive.version> @@ -349,6 +349,7 @@ <descriptor>src/main/assembly/hbase-agent.xml</descriptor> <descriptor>src/main/assembly/knox-agent.xml</descriptor> <descriptor>src/main/assembly/storm-agent.xml</descriptor> + <descriptor>src/main/assembly/plugin-yarn.xml</descriptor> <descriptor>src/main/assembly/admin-web.xml</descriptor> <descriptor>src/main/assembly/usersync.xml</descriptor> <descriptor>src/main/assembly/ranger-src.xml</descriptor> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index ca9790e..40925c0 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -441,86 +441,78 @@ public class ServiceDBStore implements ServiceStore { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.updateService()"); } - - UserSessionBase usb = ContextUtil.getCurrentUserSession(); - if (usb != null && usb.isUserAdmin()) { - XXService existing = daoMgr.getXXService().getById(service.getId()); + XXService existing = daoMgr.getXXService().getById(service.getId()); - if(existing == null) { - throw restErrorUtil.createRESTException( - "no service exists with ID=" + service.getId(), - MessageEnums.DATA_NOT_FOUND); - } - - String existingName = existing.getName(); + if(existing == null) { + throw restErrorUtil.createRESTException( + "no service exists with ID=" + service.getId(), + MessageEnums.DATA_NOT_FOUND); + } + + String existingName = existing.getName(); - boolean renamed = !StringUtils.equalsIgnoreCase(service.getName(), existingName); - - if(renamed) { - XXService newNameService = daoMgr.getXXService().findByName(service.getName()); + boolean renamed = !StringUtils.equalsIgnoreCase(service.getName(), existingName); + + if(renamed) { + XXService newNameService = daoMgr.getXXService().findByName(service.getName()); - if(newNameService != null) { - throw restErrorUtil.createRESTException("another service already exists with name '" - + service.getName() + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); - } + if(newNameService != null) { + throw restErrorUtil.createRESTException("another service already exists with name '" + + service.getName() + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); } - - Map<String, String> configs = service.getConfigs(); - Map<String, String> validConfigs = validateRequiredConfigParams( - service, configs); - if (validConfigs == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> ConfigParams cannot be null, ServiceDefDBStore.createService(" + service + ")"); - } - throw restErrorUtil.createRESTException( - "ConfigParams cannot be null.", - MessageEnums.ERROR_CREATING_OBJECT); - } - service = svcService.update(service); - XXService xUpdService = daoMgr.getXXService().getById(service.getId()); - - List<XXServiceConfigMap> dbConfigMaps = daoMgr.getXXServiceConfigMap().findByServiceId(service.getId()); - for(XXServiceConfigMap dbConfigMap : dbConfigMaps) { - daoMgr.getXXServiceConfigMap().remove(dbConfigMap); + } + + Map<String, String> configs = service.getConfigs(); + Map<String, String> validConfigs = validateRequiredConfigParams( + service, configs); + if (validConfigs == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ConfigParams cannot be null, ServiceDefDBStore.createService(" + service + ")"); } + throw restErrorUtil.createRESTException( + "ConfigParams cannot be null.", + MessageEnums.ERROR_CREATING_OBJECT); + } + service = svcService.update(service); + XXService xUpdService = daoMgr.getXXService().getById(service.getId()); + + List<XXServiceConfigMap> dbConfigMaps = daoMgr.getXXServiceConfigMap().findByServiceId(service.getId()); + for(XXServiceConfigMap dbConfigMap : dbConfigMaps) { + daoMgr.getXXServiceConfigMap().remove(dbConfigMap); + } + + VXUser vXUser = null; + XXServiceConfigMapDao xConfMapDao = daoMgr.getXXServiceConfigMap(); + for (Entry<String, String> configMap : validConfigs.entrySet()) { + String configKey = configMap.getKey(); + String configValue = configMap.getValue(); - VXUser vXUser = null; - XXServiceConfigMapDao xConfMapDao = daoMgr.getXXServiceConfigMap(); - for (Entry<String, String> configMap : validConfigs.entrySet()) { - String configKey = configMap.getKey(); - String configValue = configMap.getValue(); - - if(StringUtils.equalsIgnoreCase(configKey, "username")) { - String userName = stringUtil.getValidUserName(configValue); - XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); - if (xxUser != null) { - vXUser = xUserService.populateViewBean(xxUser); - } else { - vXUser = new VXUser(); - vXUser.setName(userName); - vXUser.setUserSource(RangerCommonEnums.USER_EXTERNAL); - vXUser = xUserMgr.createXUser(vXUser); - } + if(StringUtils.equalsIgnoreCase(configKey, "username")) { + String userName = stringUtil.getValidUserName(configValue); + XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); + if (xxUser != null) { + vXUser = xUserService.populateViewBean(xxUser); + } else { + vXUser = new VXUser(); + vXUser.setName(userName); + vXUser.setUserSource(RangerCommonEnums.USER_EXTERNAL); + vXUser = xUserMgr.createXUser(vXUser); } - - XXServiceConfigMap xConfMap = new XXServiceConfigMap(); - xConfMap = (XXServiceConfigMap) rangerAuditFields.populateAuditFields(xConfMap, xUpdService); - xConfMap.setServiceId(service.getId()); - xConfMap.setConfigkey(configKey); - xConfMap.setConfigvalue(configValue); - xConfMap = xConfMapDao.create(xConfMap); } - RangerService updService = svcService.getPopulatedViewObject(xUpdService); - dataHistService.createObjectDataHistory(updService, RangerDataHistService.ACTION_UPDATE); - return updService; - } else { - LOG.debug("User id : " + usb.getUserId() + " doesn't have admin access to update repository."); - throw restErrorUtil.createRESTException( - "Sorry, you don't have permission to perform the operation", - MessageEnums.OPER_NOT_ALLOWED_FOR_ENTITY); + XXServiceConfigMap xConfMap = new XXServiceConfigMap(); + xConfMap = (XXServiceConfigMap) rangerAuditFields.populateAuditFields(xConfMap, xUpdService); + xConfMap.setServiceId(service.getId()); + xConfMap.setConfigkey(configKey); + xConfMap.setConfigvalue(configValue); + xConfMap = xConfMapDao.create(xConfMap); } + + RangerService updService = svcService.getPopulatedViewObject(xUpdService); + dataHistService.createObjectDataHistory(updService, RangerDataHistService.ACTION_UPDATE); + + return updService; } @Override @@ -619,13 +611,7 @@ public class ServiceDBStore implements ServiceStore { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.updatePolicy(" + policy + ")"); } - UserSessionBase currentUserSession = ContextUtil - .getCurrentUserSession(); - if (currentUserSession == null) { - throw restErrorUtil.createRESTException("Policy updation not " - + "allowed",MessageEnums.OPER_NO_PERMISSION); - } - + RangerPolicy existing = getPolicy(policy.getId()); if(existing == null) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 8608054..2a71527 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -111,6 +111,8 @@ public class ServiceREST { try { ret = svcStore.createServiceDef(serviceDef); } catch(Exception excp) { + LOG.error("createServiceDef(" + serviceDef + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -135,6 +137,8 @@ public class ServiceREST { try { ret = svcStore.updateServiceDef(serviceDef); } catch(Exception excp) { + LOG.error("updateServiceDef(" + serviceDef + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -157,6 +161,8 @@ public class ServiceREST { try { svcStore.deleteServiceDef(id); } catch(Exception excp) { + LOG.error("deleteServiceDef(" + id + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -178,6 +184,8 @@ public class ServiceREST { try { ret = svcStore.getServiceDef(id); } catch(Exception excp) { + LOG.error("getServiceDef(" + id + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -205,6 +213,8 @@ public class ServiceREST { try { ret = svcStore.getServiceDefByName(name); } catch(Exception excp) { + LOG.error("getServiceDefByName(" + name + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -234,6 +244,8 @@ public class ServiceREST { try { ret = svcStore.getServiceDefs(filter); } catch(Exception excp) { + LOG.error("getServiceDefs() failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -259,6 +271,8 @@ public class ServiceREST { try { ret = svcStore.createService(service); } catch(Exception excp) { + LOG.error("createService(" + service + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -283,6 +297,8 @@ public class ServiceREST { try { ret = svcStore.updateService(service); } catch(Exception excp) { + LOG.error("updateService(" + service + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -305,6 +321,8 @@ public class ServiceREST { try { svcStore.deleteService(id); } catch(Exception excp) { + LOG.error("deleteService(" + id + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -326,6 +344,8 @@ public class ServiceREST { try { ret = svcStore.getService(id); } catch(Exception excp) { + LOG.error("getService(" + id + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -353,6 +373,8 @@ public class ServiceREST { try { ret = svcStore.getServiceByName(name); } catch(Exception excp) { + LOG.error("getServiceByName(" + name + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -382,6 +404,8 @@ public class ServiceREST { try { ret = svcStore.getServices(filter); } catch(Exception excp) { + LOG.error("getServices() failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -407,6 +431,8 @@ public class ServiceREST { ret = new Long(services == null ? 0 : services.size()); } catch(Exception excp) { + LOG.error("countServices() failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -430,6 +456,8 @@ public class ServiceREST { try { ret = serviceMgr.validateConfig(service); } catch(Exception excp) { + LOG.error("validateConfig(" + service + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -453,7 +481,7 @@ public class ServiceREST { try { ret = serviceMgr.lookupResource(serviceName,context); } catch(Exception excp) { - LOG.error("lookupResource() failed", excp); + LOG.error("lookupResource(" + serviceName + ", " + context + ") failed", excp); throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -626,7 +654,7 @@ public class ServiceREST { } catch(WebApplicationException excp) { throw excp; } catch(Exception excp) { - LOG.error("grantAccess() failed", excp); + LOG.error("grantAccess(" + serviceName + ", " + grantRequest + ") failed", excp); throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -713,7 +741,7 @@ public class ServiceREST { } catch(WebApplicationException excp) { throw excp; } catch(Exception excp) { - LOG.error("revokeAccess() failed", excp); + LOG.error("revokeAccess(" + serviceName + ", " + revokeRequest + ") failed", excp); throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -741,6 +769,8 @@ public class ServiceREST { try { ret = svcStore.createPolicy(policy); } catch(Exception excp) { + LOG.error("createPolicy(" + policy + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -764,6 +794,8 @@ public class ServiceREST { try { ret = svcStore.updatePolicy(policy); } catch(Exception excp) { + LOG.error("updatePolicy(" + policy + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -785,6 +817,8 @@ public class ServiceREST { try { svcStore.deletePolicy(id); } catch(Exception excp) { + LOG.error("deletePolicy(" + id + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -806,6 +840,8 @@ public class ServiceREST { try { ret = svcStore.getPolicy(id); } catch(Exception excp) { + LOG.error("getPolicy(" + id + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -835,6 +871,8 @@ public class ServiceREST { try { ret = svcStore.getPolicies(filter); } catch(Exception excp) { + LOG.error("getPolicies() failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -860,6 +898,8 @@ public class ServiceREST { ret = new Long(services == null ? 0 : services.size()); } catch(Exception excp) { + LOG.error("countPolicies() failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -885,6 +925,8 @@ public class ServiceREST { try { ret = svcStore.getServicePolicies(serviceId, filter); } catch(Exception excp) { + LOG.error("getServicePolicies(" + serviceId + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -914,6 +956,8 @@ public class ServiceREST { try { ret = svcStore.getServicePolicies(serviceName, filter); } catch(Exception excp) { + LOG.error("getServicePolicies(" + serviceName + ") failed", excp); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -951,6 +995,8 @@ public class ServiceREST { logMsg = "Returning " + (ret.getPolicies() != null ? ret.getPolicies().size() : 0) + " policies. Policy version=" + ret.getPolicyVersion(); } } catch(Exception excp) { + LOG.error("getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ") failed", excp); + httpCode = HttpServletResponse.SC_BAD_REQUEST; logMsg = excp.getMessage(); } finally { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c0bb7ecc/src/main/assembly/plugin-yarn.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/plugin-yarn.xml b/src/main/assembly/plugin-yarn.xml new file mode 100644 index 0000000..20a0eeb --- /dev/null +++ b/src/main/assembly/plugin-yarn.xml @@ -0,0 +1,151 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>yarn-plugin</id> + <formats> + <format>tar.gz</format> + <format>zip</format> + </formats> + <baseDirectory>${project.name}-${project.version}-yarn-plugin</baseDirectory> + <includeBaseDirectory>true</includeBaseDirectory> + <moduleSets> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <outputDirectory>/lib</outputDirectory> + <unpack>false</unpack> + <includes> + <include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include> + <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-common-plus:jar:${hadoop-common.version}</include> + <include>com.google.code.gson:gson</include> + <include>org.eclipse.persistence:eclipselink</include> + <include>org.eclipse.persistence:javax.persistence</include> + <include>commons-collections:commons-collections</include> + <include>com.sun.jersey:jersey-bundle</include> + <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> + <include>com.google.guava:guava:jar:${guava.version}</include> + </includes> + </dependencySet> + <dependencySet> + <outputDirectory>/install/lib</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <includes> + <include>commons-cli:commons-cli</include> + <include>commons-collections:commons-collections</include> + <include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include> + <include>commons-lang:commons-lang:jar:${commons.lang.version}</include> + <include>commons-logging:commons-logging</include> + <include>com.google.guava:guava:jar:${guava.version}</include> + <include>org.hamcrest:hamcrest-all</include> + <include>junit:junit</include> + <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> + <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-auth:jar:${hadoop-common.version}</include> + <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include> + <include>org.apache.ranger:credentialbuilder</include> + </includes> + </dependencySet> + </dependencySets> + <outputDirectory>/lib</outputDirectory> + </binaries> + <includes> + <include>security_plugins.ranger-plugins-audit:ranger-plugins-audit</include> + <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include> + <include>security_plugins.ranger-plugins-impl:ranger-plugins-impl</include> + <include>security_plugins.ranger-plugins-common:ranger-plugins-common</include> + <include>security_plugins.ranger-yarn-plugin:ranger-yarn-plugin</include> + </includes> + </moduleSet> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <outputDirectory>/install/lib</outputDirectory> + <unpack>false</unpack> + </binaries> + <includes> + <include>security_plugins.ranger-plugins-installer:ranger-plugins-installer</include> + <include>org.apache.ranger:credentialbuilder</include> + </includes> + </moduleSet> + </moduleSets> + <fileSets> + <!-- conf.templates for enable --> + <fileSet> + <outputDirectory>/install/conf.templates/enable</outputDirectory> + <directory>plugin-yarn/conf</directory> + <excludes> + <exclude>*.sh</exclude> + </excludes> + <fileMode>700</fileMode> + </fileSet> + <fileSet> + <outputDirectory>/install/conf.templates/disable</outputDirectory> + <directory>plugin-yarn/disable-conf</directory> + <fileMode>700</fileMode> + </fileSet> + <fileSet> + <outputDirectory>/install/conf.templates/default</outputDirectory> + <directory>plugin-yarn/template</directory> + <fileMode>700</fileMode> + </fileSet> + <!-- version file --> + <fileSet> + <outputDirectory>/</outputDirectory> + <directory>${project.build.outputDirectory}</directory> + <includes> + <include>version</include> + </includes> + <fileMode>444</fileMode> + </fileSet> + </fileSets> + <!-- enable/disable script for Plugin --> + <files> + <file> + <source>agents-common/scripts/enable-agent.sh</source> + <outputDirectory>/</outputDirectory> + <destName>enable-yarn-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>agents-common/scripts/enable-agent.sh</source> + <outputDirectory>/</outputDirectory> + <destName>disable-yarn-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>plugin-yarn/scripts/install.properties</source> + <outputDirectory>/</outputDirectory> + <destName>install.properties</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>plugin-yarn/scripts/yarn-plugin-install.properties</source> + <outputDirectory>/</outputDirectory> + <destName>yarn-plugin-install.properties</destName> + <fileMode>755</fileMode> + </file> + </files> +</assembly>
