Repository: incubator-eagle Updated Branches: refs/heads/master 1a3b9b086 -> d394a8e34
EAGLE-61 classification supports secured cluster connection & enable Ldap authentication https://issues.apache.org/jira/browse/EAGLE-61 Author: qingwen, zhao<qingwz...@apache.org> Reviewer: Ralph, su Closes #111 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/d394a8e3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/d394a8e3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/d394a8e3 Branch: refs/heads/master Commit: d394a8e34ddb758f1f6f96bf40346d176859628b Parents: 1a3b9b0 Author: Zhao, Qingwen <qingwz...@ebay.com> Authored: Fri Mar 4 15:15:24 2016 +0800 Committer: Zhao, Qingwen <qingwz...@ebay.com> Committed: Fri Mar 4 15:15:24 2016 +0800 ---------------------------------------------------------------------- eagle-assembly/src/main/bin/eagle-service.sh | 6 ++ .../src/main/bin/eagle-topology-init.sh | 4 +- eagle-assembly/src/main/conf/eagle-service.conf | 6 ++ eagle-assembly/src/main/conf/ldap.properties | 25 ++++++ .../lib/EAGLE/package/scripts/actions.py | 6 +- .../eagle/log4j/kafka/KafkaLog4jAppender.scala | 7 +- .../eagle/log4j/kafka/TestGenericLogKeyer.scala | 58 ++++++++------ .../log4j/kafka/TestKafkaLog4jAppender.scala | 25 +++--- .../resolver/MetadataAccessConfigRepo.java | 59 ++++++++++++++ .../eagle/security/util/HadoopSecurityUtil.java | 46 +++++++++++ .../eagle/security/util/LogParseUtil.java | 1 + .../hbase/parse/HbaseAuditLogParser.java | 3 +- .../hbase/HbaseMetadataBrowseWebResource.java | 12 +-- .../hbase/dao/HbaseMetadataAccessConfig.java | 43 ----------- .../dao/HbaseMetadataAccessConfigDAOImpl.java | 61 --------------- .../hbase/dao/HbaseMetadataDAOImpl.java | 14 ++-- .../hbase/resolver/HbaseMetadataResolver.java | 9 ++- .../service/security/hdfs/HDFSFileSystem.java | 81 ++++++++------------ .../security/hdfs/HDFSResourceAccessConfig.java | 45 ----------- .../security/hdfs/HDFSResourceConstants.java | 2 +- .../security/hdfs/HDFSResourceUtils.java | 63 --------------- .../hdfs/resolver/HDFSResourceResolver.java | 32 ++++---- .../rest/HDFSResourceWebRequestValidator.java | 13 +++- .../hdfs/rest/HDFSResourceWebResource.java | 33 ++++---- .../security/hdfs/test/HDFSFileSystemTest.java | 34 -------- .../hdfs/test/HDFSResourceAccessConfigTest.java | 30 +++++--- .../hdfs/test/HDFSResourceConfigAccessTest.java | 34 -------- .../hdfs/test/HDFSResourceResolverTest.java | 36 --------- .../HDFSResourceSensitivityServiceTest.java | 42 ---------- .../security/hdfs/test/HDFSResourceWebTest.java | 35 --------- .../security/auth/AuthoritiesPopulator.java | 62 +++++++++++++++ .../src/main/resources/eagleSecurity.xml | 73 +++++++++--------- 32 files changed, 414 insertions(+), 586 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-assembly/src/main/bin/eagle-service.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/bin/eagle-service.sh b/eagle-assembly/src/main/bin/eagle-service.sh index cdf097e..d104f73 100755 --- a/eagle-assembly/src/main/bin/eagle-service.sh +++ b/eagle-assembly/src/main/bin/eagle-service.sh @@ -50,6 +50,7 @@ fi EAGLE_SERVICE_CONF="eagle-service.conf" +EAGLE_LDAP_CONF="ldap.properties" # Always copy conf/eagle-service.properties to lib/tomcat/webapps/eagle-service/WEB-INF/classes/application.conf before starting if [ ! -e ${EAGLE_HOME}/conf/${EAGLE_SERVICE_CONF} ] @@ -59,6 +60,11 @@ then fi cp -f $EAGLE_HOME/conf/$EAGLE_SERVICE_CONF ${EAGLE_HOME}/lib/tomcat/webapps/eagle-service/WEB-INF/classes/application.conf +if [ -e ${EAGLE_HOME}/conf/${EAGLE_LDAP_CONF} ] +then + cp -f $EAGLE_HOME/conf/$EAGLE_LDAP_CONF ${EAGLE_HOME}/lib/tomcat/webapps/eagle-service/WEB-INF/classes/ +fi + case $1 in "start") echo "Starting eagle service ..." http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-assembly/src/main/bin/eagle-topology-init.sh ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/bin/eagle-topology-init.sh b/eagle-assembly/src/main/bin/eagle-topology-init.sh index 5d82efd..c326e38 100755 --- a/eagle-assembly/src/main/bin/eagle-topology-init.sh +++ b/eagle-assembly/src/main/bin/eagle-topology-init.sh @@ -33,9 +33,9 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a echo "" echo "Importing applications for sample site ..." -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"; -d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hdfsAuditLog"}, "enabled": true, "config" : "{\"hdfsEndpoint\":\"hdfs://sandbox.hortonworks.com:8020\"}"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"; -d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hdfsAuditLog"}, "enabled": true, "config" : "{\"fs.defaultFS\":\"hdfs://sandbox.hortonworks.com:8020\"}"}]' -curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"; -d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hbaseSecurityLog"}, "enabled": true, "config" : "{\"zkClientPort\":\"2181\", \"zkQuorum\":\"localhost\"}"}]' +curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"; -d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hbaseSecurityLog"}, "enabled": true, "config" : "{\"hbase.zookeeper.property.clientPort\":\"2181\", \"hbase.zookeeper.quorum\":\"localhost\"}"}]' curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"; -d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hiveQueryLog"}, "enabled": true, "config":"{\"accessType\":\"metastoredb_jdbc\",\"password\":\"hive\",\"user\":\"hive\",\"jdbcDriverClassName\":\"com.mysql.jdbc.Driver\",\"jdbcUrl\":\"jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true\"}"}]' http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-assembly/src/main/conf/eagle-service.conf ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/eagle-service.conf b/eagle-assembly/src/main/conf/eagle-service.conf index 229458c..8e2b96a 100644 --- a/eagle-assembly/src/main/conf/eagle-service.conf +++ b/eagle-assembly/src/main/conf/eagle-service.conf @@ -32,5 +32,11 @@ eagle{ # hbase configuration: zookeeper.znode.parent # default is "/hbase" zookeeper-znode-parent="/hbase-unsecure" + + # eagle web login profile: [sandbox, default] + # default is sandbox + springActiveProfile="sandbox" + + audit-enabled=false } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-assembly/src/main/conf/ldap.properties ---------------------------------------------------------------------- diff --git a/eagle-assembly/src/main/conf/ldap.properties b/eagle-assembly/src/main/conf/ldap.properties new file mode 100644 index 0000000..9f31a43 --- /dev/null +++ b/eagle-assembly/src/main/conf/ldap.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +ldap.server= +ldap.username= +ldap.password= +ldap.user.searchBase= +ldap.user.searchPattern= +ldap.user.groupSearchBase= +acl.adminRole= +acl.defaultRole= \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/actions.py ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/actions.py b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/actions.py index ffd3c63..053eeee 100755 --- a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/actions.py +++ b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/actions.py @@ -64,7 +64,7 @@ def eagle_topology_exec(action="start"): def eagle_hive_topology_exec(action="start"): import params - main_class="eagle.security.hive.jobrunning.HiveJobRunningMonitoringMain" + main_class="org.apache.eagle.security.hive.jobrunning.HiveJobRunningMonitoringMain" topology_name=format("{eagle_site}-hiveQueryRunningTopology") config_file=format("{eagle_conf}/{eagle_site}-hiveQueryLog-application.conf") eagle_topology_shell=format("{eagle_bin}/eagle-topology.sh") @@ -89,7 +89,7 @@ def eagle_hive_topology_exec(action="start"): def eagle_hdfs_topology_exec(action="start"): import params - main_class="eagle.security.auditlog.HdfsAuditLogProcessorMain" + main_class="org.apache.eagle.security.auditlog.HdfsAuditLogProcessorMain" topology_name=format("{eagle_site}-hdfsAuditLog-topology") config_file=format("{eagle_conf}/{eagle_site}-hdfsAuditLog-application.conf") @@ -114,7 +114,7 @@ def eagle_hdfs_topology_exec(action="start"): def eagle_userprofile_topology_exec(action="start"): import params - main_class="eagle.security.userprofile.UserProfileDetectionMain" + main_class="org.apache.eagle.security.userprofile.UserProfileDetectionMain" topology_name=format("{eagle_site}-userprofile-topology") config_file=format("{eagle_conf}/{eagle_site}-userprofile-topology.conf") http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala index ee09294..4b5dee1 100644 --- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala +++ b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala @@ -101,8 +101,6 @@ class KafkaLog4jAppender extends AppenderSkeleton with Logging { props.put("serializer.class", serializerClass) props.put("batch.num.messages",batchSize.toString) - if(keyPattern != null) props.put("keyPattern", keyPattern) - //These have default values in ProducerConfig and AsyncProducerConfig. We don't care if they're not specified if(producerType != null) props.put("producer.type", producerType) if(compressionCodec != null) props.put("compression.codec", compressionCodec) @@ -114,8 +112,11 @@ class KafkaLog4jAppender extends AppenderSkeleton with Logging { producer = new Producer[String, String](config) + val extraProps = new Properties() + if(keyPattern != null) extraProps.put("keyPattern", keyPattern) + if(keyClass != null){ - keyer = Utils.createObject[Keyer](keyClass,props) + keyer = Utils.createObject[Keyer](keyClass, extraProps) LogLog.debug("Instantiated Key class " + keyClass) } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala index 840dfe7..7e1e1ff 100644 --- a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala +++ b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala @@ -20,27 +20,37 @@ package org.apache.eagle.log4j.kafka import java.util.Properties import org.apache.eagle.log4j.kafka.hadoop.GenericLogKeyer -import org.scalatest.{FlatSpec, Matchers} - -abstract class TestGenericLogKeyer extends FlatSpec with Matchers { - - val hdfsMsg = "2015-07-31 01:54:35,161 INFO FSNamesystem.audit: allowed=true ugi=root (auth:TOKEN) ip=/10.0.0.1 cmd=open src=/tmp/private dst=null perm=null" - val props = new Properties() - props.put("keyPattern", "ugi=(\\w+)[@\\s+]") - props.put("keyPattern2", "user=(\\w+),\\s+") - val test = new GenericLogKeyer(props) - var keyVal = test.getKey(hdfsMsg) - println(keyVal) - - val hbaseMsg = "2015-11-06 13:14:00,741 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)" - props.put("keyPattern", "user=(\\w+),\\s+") - keyVal = test.getKey(hbaseMsg) - println(keyVal) - - //props.put("keyPattern", "user=(\\w+),\\s+") - val props1 = new Properties() - val test1 = new GenericLogKeyer(props1) - keyVal = test1.getKey(hbaseMsg) - println(keyVal) - -} +import org.junit.Test + + + +class TestGenericLogKeyer { + + @Test def testHdfsAuditMsgParse() { + val hdfsAuditMsg = "2015-07-31 01:54:35,161 INFO FSNamesystem.audit: allowed=true ugi=root (auth:TOKEN) ip=/10.0.0.1 cmd=open src=/tmp/private dst=null perm=null" + val props = new Properties() + props.put("keyPattern", "ugi=(\\w+)[@\\s+]") + props.put("keyPattern2", "user=(\\w+),\\s+") + val test = new GenericLogKeyer(props) + var keyVal = test.getKey(hdfsAuditMsg) + println(keyVal) + } + + @Test def testHbaseSecurityMsgParse() { + val hbaseMsg = "2015-11-06 13:14:00,741 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)" + val props = new Properties() + props.put("keyPattern", "user=(\\w+),\\s+") + val test = new GenericLogKeyer(props) + val keyVal = test.getKey(hbaseMsg) + + } + + @Test def testHdfsSecurityMsgParse() { + val hdfsSecurityMsg = "2015-12-24 03:17:47,019 INFO SecurityLogger.org.apache.hadoop.security.authorize.ServiceAuthorizationManager: Authorization successful for hive (auth:SIMPLE) for protocol=interface org.apache.hadoop.hdfs.protocol.ClientProtocol" + val props = new Properties() + props.put("keyPattern", "for\\s+(\\w+)[@\\s+]") + val test2 = new GenericLogKeyer(props) + val keyVal = test2.getKey(hdfsSecurityMsg) + println(keyVal) + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala index 3bd9e0e..20d5ff5 100644 --- a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala +++ b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala @@ -18,16 +18,17 @@ package org.apache.eagle.log4j.kafka -import org.scalatest.{Matchers, FlatSpec} +import org.junit.Test -abstract class TestKafkaLog4jAppender extends FlatSpec with Matchers { - - val test = new KafkaLog4jAppender(); - test.topic = "sandbox_hdfs_audit_log" - test.brokerList = "sandbox.hortonworks.com:6667" - test.keyClass = "eagle.log4j.kafka.hadoop.AuditLogKeyer" - test.keyPattern = "user=(\\w+),\\s+" - test.producerType = "async" - test.activateOptions() - -} +class TestKafkaLog4jAppender { + + @Test def test() { + val test = new KafkaLog4jAppender(); + test.topic = "sandbox_hdfs_audit_log" + test.brokerList = "sandbox.hortonworks.com:6667" + test.keyClass = "org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer" + test.keyPattern = "user=(\\w+),\\s+" + test.producerType = "async" + test.activateOptions() + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java new file mode 100644 index 0000000..f3fce8d --- /dev/null +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.security.resolver; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.eagle.alert.entity.SiteApplicationServiceEntity; +import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; +import org.apache.eagle.policy.common.Constants; +import org.apache.eagle.service.generic.GenericEntityServiceResource; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + + +public class MetadataAccessConfigRepo { + private static Logger LOG = LoggerFactory.getLogger(MetadataAccessConfigRepo.class); + + public Configuration getConfig(String application, String siteId) throws Exception { + + GenericEntityServiceResource resource = new GenericEntityServiceResource(); + String queryFormat = Constants.SITE_APPLICATION_SERVICE_ENDPOINT_NAME + "[@application=\"%s\" AND @site=\"%s\"]{*}"; + GenericServiceAPIResponseEntity ret = resource.search(String.format(queryFormat, application, siteId), null, null,Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); + List<SiteApplicationServiceEntity> list = (List<SiteApplicationServiceEntity>) ret.getObj(); + if (list == null || list.size() == 0) + throw new Exception("Config is empty for site=" + siteId +" application=" + application + "."); + + ObjectMapper mapper = new ObjectMapper(); + Map<String, String> configMap = mapper.readValue(list.get(0).getConfig(), Map.class); + return convert(configMap); + } + + private Configuration convert(Map<String, String> configMap) throws Exception { + Configuration config = new Configuration(); + for (Map.Entry<String, String> entry : configMap.entrySet()) { + config.set(entry.getKey(), entry.getValue()); + } + return config; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/HadoopSecurityUtil.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/HadoopSecurityUtil.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/HadoopSecurityUtil.java new file mode 100644 index 0000000..b0e53f8 --- /dev/null +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/HadoopSecurityUtil.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.security.util; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; + + +/** + * This class provides util methods for Eagle connector communicating + * with secured cluster. + */ +public class HadoopSecurityUtil { + + public static final String EAGLE_KEYTAB_FILE_KEY = "eagle.keytab.file"; + public static final String EAGLE_USER_NAME_KEY = "eagle.kerberos.principal"; + + public static void login(Configuration kConfig) throws IOException { + if (kConfig.get(EAGLE_KEYTAB_FILE_KEY) == null || kConfig.get(EAGLE_USER_NAME_KEY) == null) return; + + kConfig.setBoolean("hadoop.security.authorization", true); + kConfig.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(kConfig); + UserGroupInformation.loginUserFromKeytab(kConfig.get(EAGLE_USER_NAME_KEY), kConfig.get(EAGLE_KEYTAB_FILE_KEY)); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/LogParseUtil.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/LogParseUtil.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/LogParseUtil.java index d1cb8b5..0700cd2 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/LogParseUtil.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/LogParseUtil.java @@ -27,6 +27,7 @@ public class LogParseUtil { * 1)u...@apd.xyz.com * 2)hadoop/123.dc1.xyz....@xyz.com (auth:KERBEROS) * 3)hadoop (auth:KERBEROS) + * 4)hadoop */ public static String parseUserFromUGI(String newUgi) { if(newUgi == null) return null; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java index 89e57fd..6fdb03f 100644 --- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java +++ b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java @@ -25,6 +25,7 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.security.util.LogParseUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +76,7 @@ public class HbaseAuditLogParser implements Serializable { } ret.scope = scope; ret.action = auditMap.get(ACTION); - ret.user = auditMap.get(USER); + ret.user = LogParseUtil.parseUserFromUGI(auditMap.get(USER)); ret.request = auditMap.get(REQUEST); ret.timestamp = DateTimeUtil.humanDateToMilliseconds(auditMap.get(LOGDATE)); return ret; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java index 5dc98ca..b15b15b 100644 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java @@ -17,10 +17,10 @@ package org.apache.eagle.service.security.hbase; import org.apache.eagle.security.entity.HbaseResourceEntity; +import org.apache.eagle.security.resolver.MetadataAccessConfigRepo; import org.apache.eagle.service.common.EagleExceptionWrapper; -import org.apache.eagle.service.security.hbase.dao.HbaseMetadataAccessConfig; -import org.apache.eagle.service.security.hbase.dao.HbaseMetadataAccessConfigDAOImpl; import org.apache.eagle.service.security.hbase.dao.HbaseMetadataDAOImpl; +import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +34,8 @@ public class HbaseMetadataBrowseWebResource { private static Logger LOG = LoggerFactory.getLogger(HbaseMetadataBrowseWebResource.class); private HbaseSensitivityResourceService dao = new HbaseSensitivityResourceService(); private Map<String, Map<String, String>> maps = dao.getAllHbaseSensitivityMap(); + private MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo(); + final public static String HBASE_APPLICATION = "hbaseSecurityLog"; @Path("/namespaces") @GET @@ -44,7 +46,7 @@ public class HbaseMetadataBrowseWebResource { List<HbaseResourceEntity> values = new ArrayList<>(); HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse(); try { - HbaseMetadataAccessConfig config = new HbaseMetadataAccessConfigDAOImpl().getConfig(site); + Configuration config = repo.getConfig(HBASE_APPLICATION, site); HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config); namespaces = dao.getNamespaces(); @@ -72,7 +74,7 @@ public class HbaseMetadataBrowseWebResource { List<String> tables = null; List<HbaseResourceEntity> values = new ArrayList<>(); try { - HbaseMetadataAccessConfig config = new HbaseMetadataAccessConfigDAOImpl().getConfig(site); + Configuration config = repo.getConfig(HBASE_APPLICATION, site); HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config); tables = dao.getTables(namespace); }catch(Exception ex){ @@ -102,7 +104,7 @@ public class HbaseMetadataBrowseWebResource { List<String> columns = null; List<HbaseResourceEntity> values = new ArrayList<>(); try { - HbaseMetadataAccessConfig config = new HbaseMetadataAccessConfigDAOImpl().getConfig(site); + Configuration config = repo.getConfig(HBASE_APPLICATION, site); HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config); String tableName = String.format("%s:%s", namespace, table); columns = dao.getColumnFamilies(tableName); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfig.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfig.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfig.java deleted file mode 100644 index 2e91974..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.service.security.hbase.dao; - - -public class HbaseMetadataAccessConfig { - private String zkQuorum; - - public String getZkClientPort() { - return zkClientPort; - } - - public void setZkClientPort(String zkClientPort) { - this.zkClientPort = zkClientPort; - } - - private String zkClientPort; - - public String getZkQuorum() { - return zkQuorum; - } - - public void setZkQuorum(String zkQuorum) { - this.zkQuorum = zkQuorum; - } -} - http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfigDAOImpl.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfigDAOImpl.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfigDAOImpl.java deleted file mode 100644 index 152facb..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataAccessConfigDAOImpl.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.service.security.hbase.dao; - - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.eagle.alert.entity.SiteApplicationServiceEntity; -import org.apache.eagle.log.entity.ListQueryAPIResponseEntity; -import org.apache.eagle.policy.common.Constants; -import org.apache.eagle.security.util.BadMetadataAccessConfigException; -import org.apache.eagle.service.generic.ListQueryResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -public class HbaseMetadataAccessConfigDAOImpl { - private static Logger LOG = LoggerFactory.getLogger(HbaseMetadataAccessConfigDAOImpl.class); - - private HbaseMetadataAccessConfig convert(String config){ - ObjectMapper mapper = new ObjectMapper(); - HbaseMetadataAccessConfig c = null; - try { - c = mapper.readValue(config, HbaseMetadataAccessConfig.class); - }catch(Exception ex){ - LOG.error("config block could be broken", ex); - throw new BadMetadataAccessConfigException(ex); - } - return c; - } - - - public HbaseMetadataAccessConfig getConfig(String site) throws Exception{ - ListQueryResource resource = new ListQueryResource(); - /* parameters are: query, startTime, endTime, pageSzie, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, - * parallel, metricName*/ - String queryFormat = Constants.SITE_APPLICATION_SERVICE_ENDPOINT_NAME + "[@application=\"hbaseSecurityLog\" AND @site=\"%s\"]{*}"; - ListQueryAPIResponseEntity ret = resource.listQuery(String.format(queryFormat, site), null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null); - List<SiteApplicationServiceEntity> list = (List<SiteApplicationServiceEntity>) ret.getObj(); - if(list == null || list.size() ==0) - throw new BadMetadataAccessConfigException("config is empty for site " + site); - return convert(list.get(0).getConfig()); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java index 6f735da..5d42ba8 100644 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java @@ -19,6 +19,7 @@ package org.apache.eagle.service.security.hbase.dao; +import org.apache.eagle.security.util.HadoopSecurityUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -33,16 +34,17 @@ public class HbaseMetadataDAOImpl { private final static Logger LOG = LoggerFactory.getLogger(HbaseMetadataDAOImpl.class); private Configuration hBaseConfiguration; - private HbaseMetadataAccessConfig config; - public HbaseMetadataDAOImpl(HbaseMetadataAccessConfig config) { - this.config = config; + + public HbaseMetadataDAOImpl(Configuration config) { this.hBaseConfiguration = HBaseConfiguration.create(); - this.hBaseConfiguration.set("hbase.zookeeper.quorum", this.config.getZkQuorum()); - this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", this.config.getZkClientPort()); + this.hBaseConfiguration.addResource(config); + //this.hBaseConfiguration.set("hbase.zookeeper.quorum", this.config.getZkQuorum()); + //this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", this.config.getZkClientPort()); } private HBaseAdmin getHBaseAdmin() throws IOException { + HadoopSecurityUtil.login(hBaseConfiguration); return new HBaseAdmin(this.hBaseConfiguration); } @@ -91,4 +93,4 @@ public class HbaseMetadataDAOImpl { } } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java index a60447b..c7b1b5c 100644 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java @@ -19,19 +19,21 @@ package org.apache.eagle.service.security.hbase.resolver; +import org.apache.eagle.security.resolver.MetadataAccessConfigRepo; import org.apache.eagle.service.alert.resolver.AttributeResolvable; import org.apache.eagle.service.alert.resolver.AttributeResolveException; import org.apache.eagle.service.alert.resolver.BadAttributeResolveRequestException; import org.apache.eagle.service.alert.resolver.GenericAttributeResolveRequest; -import org.apache.eagle.service.security.hbase.dao.HbaseMetadataAccessConfig; -import org.apache.eagle.service.security.hbase.dao.HbaseMetadataAccessConfigDAOImpl; +import org.apache.eagle.service.security.hbase.HbaseMetadataBrowseWebResource; import org.apache.eagle.service.security.hbase.dao.HbaseMetadataDAOImpl; +import org.apache.hadoop.conf.Configuration; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; + public class HbaseMetadataResolver implements AttributeResolvable<GenericAttributeResolveRequest,String> { @Override public List<String> resolve(GenericAttributeResolveRequest request) throws AttributeResolveException { @@ -40,7 +42,8 @@ public class HbaseMetadataResolver implements AttributeResolvable<GenericAttribu String[] subResources = query.split(":"); try { - HbaseMetadataAccessConfig config = new HbaseMetadataAccessConfigDAOImpl().getConfig(site); + MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo(); + Configuration config = repo.getConfig(HbaseMetadataBrowseWebResource.HBASE_APPLICATION, site); HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config); switch (subResources.length) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java index 67e4092..7a07912 100644 --- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java +++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSFileSystem.java @@ -1,13 +1,12 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,12 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.eagle.service.security.hdfs; import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.apache.eagle.security.util.HadoopSecurityUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -29,74 +30,52 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * + * * This class is responsible for all FileSystem Querying Operation - * Example + * Example * List of Directories * List of Files and It's Paths - * + * * This internally it uses Hadoop FileSystem API to List the files - * + * */ public class HDFSFileSystem { - private String hdfsEndPoint; + private Configuration config; + private static Logger LOG = LoggerFactory.getLogger(HDFSFileSystem.class); - - public HDFSFileSystem( String hdfsEndPoint ) - { - this.hdfsEndPoint = hdfsEndPoint; + + public HDFSFileSystem(Configuration hdfsConfig) { + config = hdfsConfig; } - - /** - * Creates FileSystem Object - * @param config - * @return - * @throws IOException - */ - public FileSystem getFileSystem( Configuration config ) throws IOException - { - + + public FileSystem getFileSystem() throws IOException { + HadoopSecurityUtil.login(config); return FileSystem.get(config); } - + /** * Brows the Files for the specific Path - * + * * @param filePath * @return listOfFiles - * @throws Exception + * @throws Exception */ - public List<FileStatus> browse(String filePath) throws Exception - { - LOG.info("HDFS File Path : "+filePath +" and EndPoint : "+hdfsEndPoint); + public List<FileStatus> browse(String filePath) throws Exception { FileSystem hdfsFileSystem = null; - FileStatus[] listStatus; - try { - Configuration config = createConfig(); - hdfsFileSystem = getFileSystem(config); + FileStatus[] listStatus; + try { + hdfsFileSystem = getFileSystem(); Path path = new Path(filePath); listStatus = hdfsFileSystem.listStatus( path ); } catch ( Exception ex ) { LOG.error(" Exception when browsing files for the path " +filePath , ex.getMessage() ); throw new Exception(" Exception When browsing Files in HDFS .. Message : "+ex.getMessage()); } finally { - //Close the file system + //Close the file system if( hdfsFileSystem != null ) hdfsFileSystem.close(); } return Arrays.asList(listStatus); } - - /** - * Create Config Object - * @return - */ - public Configuration createConfig() throws Exception { - Configuration config = new Configuration(); - config.set(HDFSResourceConstants.HDFS_FS_DEFAULT_NAME, this.hdfsEndPoint); - return config; - } - - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceAccessConfig.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceAccessConfig.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceAccessConfig.java deleted file mode 100644 index 74cb8b0..0000000 --- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceAccessConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.security.hdfs; - - -/** - * - * Pre Assumption - * - * - * The above config will not change .. - */ -public class HDFSResourceAccessConfig { - - private String hdfsEndpoint; - - public String getHdfsEndpoint() { - return hdfsEndpoint; - } - - public void setHdfsEndpoint(String hdfsEndpoint) { - this.hdfsEndpoint = hdfsEndpoint; - } - - @Override - public String toString() { - return "HDFSResourceAccessConfig [hdfsEndpoint=" + hdfsEndpoint + "]"; - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceConstants.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceConstants.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceConstants.java index 9892608..497afa5 100644 --- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceConstants.java +++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceConstants.java @@ -20,6 +20,6 @@ public class HDFSResourceConstants { public static final String HDFS_RESOURCE="/hdfsResource"; public static final String HDFS_FS_DEFAULT_NAME="fs.defaultFS"; - public static final String HDFS_DATA_SOURCE="hdfsAuditLog"; + public static final String HDFS_APPLICATION="hdfsAuditLog"; public static final String HDFS_RESOURCE_RESOLVE_FORMAT_HINT ="HDFS Resource must be start with /"; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceUtils.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceUtils.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceUtils.java deleted file mode 100644 index bece538..0000000 --- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/HDFSResourceUtils.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.security.hdfs; - -import java.util.List; - -import org.apache.eagle.alert.entity.SiteApplicationServiceEntity; -import org.apache.eagle.policy.common.Constants; -import org.apache.eagle.service.generic.ListQueryResource; - -import org.apache.eagle.log.entity.ListQueryAPIResponseEntity; -import com.fasterxml.jackson.databind.ObjectMapper; - - -/** - * Util API which has common methods and Service calls API - */ -public class HDFSResourceUtils { - - @SuppressWarnings("unchecked") - public static HDFSResourceAccessConfig getConfig(String siteId ) throws Exception - { - ListQueryResource resource = new ListQueryResource(); - String queryFormat = Constants.SITE_APPLICATION_SERVICE_ENDPOINT_NAME+"[@application=\""+HDFSResourceConstants.HDFS_DATA_SOURCE+"\" AND @site=\"%s\"]{*}"; - ListQueryAPIResponseEntity ret = resource.listQuery(String.format(queryFormat, siteId), null, null,Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null); - List<SiteApplicationServiceEntity> list = (List<SiteApplicationServiceEntity>) ret.getObj(); - if (list == null || list.size() == 0) - throw new Exception("Config is empty for site " + siteId +"."); - - ObjectMapper mapper = new ObjectMapper(); - HDFSResourceAccessConfig config = mapper.readValue(list.get(0).getConfig(), HDFSResourceAccessConfig.class); - return config; - } - - /** - * Not Null String Check Method - * @param input - * @return - */ - - public static boolean isNullOrEmpty( String input ) - { - if( null == input || input.length() <= 0 ) - return true; - - return false; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java index aeb99a7..71b67ec 100644 --- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java +++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java @@ -21,40 +21,42 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.eagle.security.resolver.MetadataAccessConfigRepo; import org.apache.eagle.service.alert.resolver.AttributeResolvable; import org.apache.eagle.service.alert.resolver.AttributeResolveException; import org.apache.eagle.service.alert.resolver.BadAttributeResolveRequestException; import org.apache.eagle.service.alert.resolver.GenericAttributeResolveRequest; -import org.apache.eagle.service.security.hdfs.HDFSResourceAccessConfig; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.eagle.service.security.hdfs.HDFSFileSystem; import org.apache.eagle.service.security.hdfs.HDFSResourceConstants; -import org.apache.eagle.service.security.hdfs.HDFSResourceUtils; + /** - * HDFS Resource Resolver + * HDFS Resource Resolver * - * Generic Resolver Will invoke this HDFS Resolvers + * Generic Resolver Will invoke this HDFS Resolvers * Query HINT : HDFS Resource resolve must be {\"site\":\"${site}\", \"query\"=\"{directory path}\"} */ public class HDFSResourceResolver implements AttributeResolvable<GenericAttributeResolveRequest,String> { private final static Logger LOG = LoggerFactory.getLogger(HDFSResourceResolver.class); /** * HDFS Resource Resolve API - * + * * returns listOfPaths */ @Override public List<String> resolve(GenericAttributeResolveRequest request) throws AttributeResolveException { - List<String> result = new ArrayList<>(); + List<String> result = new ArrayList<>(); + MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo(); try { - HDFSResourceAccessConfig config = HDFSResourceUtils.getConfig(request.getSite().trim()); - HDFSFileSystem fileSystem = new HDFSFileSystem(config.getHdfsEndpoint()); + Configuration config = repo.getConfig(HDFSResourceConstants.HDFS_APPLICATION, request.getSite().trim()); + HDFSFileSystem fileSystem = new HDFSFileSystem(config); String query = request.getQuery().trim(); List<FileStatus> fileStatuses = null; if(query.endsWith("/")) { @@ -70,9 +72,9 @@ public class HDFSResourceResolver implements AttributeResolvable<GenericAttribu throw new BadAttributeResolveRequestException(HDFSResourceConstants.HDFS_RESOURCE_RESOLVE_FORMAT_HINT); } } - for(FileStatus status: fileStatuses){ - result.add(status.getPath().toUri().getPath()); - } + for(FileStatus status: fileStatuses){ + result.add(status.getPath().toUri().getPath()); + } LOG.info("Successfully browsed files in HDFS ."); return result; @@ -81,15 +83,15 @@ public class HDFSResourceResolver implements AttributeResolvable<GenericAttribu throw new AttributeResolveException(e); } } - + /** * Validate the Passed Request Object - * It should have Site Id and File Path + * It should have Site Id and File Path */ @Override public void validateRequest(GenericAttributeResolveRequest request) throws BadAttributeResolveRequestException { - if(LOG.isDebugEnabled()) LOG.debug(" validating HDFS Resource Resolve request ..."); + if(LOG.isDebugEnabled()) LOG.debug(" validating HDFS Resource Resolve request ..."); String siteId = request.getSite(); if( null == siteId ) throw new BadAttributeResolveRequestException(HDFSResourceConstants.HDFS_RESOURCE_RESOLVE_FORMAT_HINT); @@ -115,7 +117,7 @@ public class HDFSResourceResolver implements AttributeResolvable<GenericAttribu return result; } /** - * + * */ @Override public Class<GenericAttributeResolveRequest> getRequestClass() { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebRequestValidator.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebRequestValidator.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebRequestValidator.java index 326eb15..14801f8 100644 --- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebRequestValidator.java +++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebRequestValidator.java @@ -16,7 +16,6 @@ */ package org.apache.eagle.service.security.hdfs.rest; -import org.apache.eagle.service.security.hdfs.HDFSResourceUtils; /** * Validate the REST API request @@ -30,9 +29,17 @@ public class HDFSResourceWebRequestValidator { * @throws Exception */ public void validate( String site, String filePath ) throws Exception { - if (HDFSResourceUtils.isNullOrEmpty(site)) + if (isNullOrEmpty(site)) throw new Exception("Invalid Request Received ... Site is Empty Or Null.."); - if (HDFSResourceUtils.isNullOrEmpty(filePath)) + if (isNullOrEmpty(filePath)) throw new Exception("Invalid Request Received ... file/Directory Path is Empty Or Null.."); } + + public static boolean isNullOrEmpty( String input ) + { + if( null == input || input.length() <= 0 ) + return true; + + return false; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java index 588979f..49e4372 100644 --- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java +++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java @@ -26,45 +26,46 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; +import org.apache.eagle.security.entity.FileStatusEntity; +import org.apache.eagle.security.resolver.MetadataAccessConfigRepo; import org.apache.eagle.service.common.EagleExceptionWrapper; -import org.apache.eagle.service.security.hdfs.HDFSResourceAccessConfig; import org.apache.eagle.service.security.hdfs.HDFSResourceConstants; import org.apache.eagle.service.security.hdfs.HDFSResourceSensitivityDataJoiner; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.eagle.service.security.hdfs.HDFSFileSystem; -import org.apache.eagle.service.security.hdfs.HDFSResourceUtils; -import org.apache.eagle.security.entity.FileStatusEntity; /** * REST Web Service to browse files and Paths in HDFS */ @Path(HDFSResourceConstants.HDFS_RESOURCE) -public class HDFSResourceWebResource +public class HDFSResourceWebResource { private static Logger LOG = LoggerFactory.getLogger(HDFSResourceWebResource.class); - - @GET - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) + + @GET + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) public HDFSResourceWebResponse getHDFSResource( @QueryParam("site") String site , @QueryParam("path") String filePath ) { LOG.info("Starting HDFS Resource Browsing. Query Parameters ==> Site :"+site+" Path : "+filePath ); - HDFSResourceWebResponse response = new HDFSResourceWebResponse(); - HDFSResourceWebRequestValidator validator = new HDFSResourceWebRequestValidator(); - List<FileStatusEntity> result = new ArrayList<>(); + HDFSResourceWebResponse response = new HDFSResourceWebResponse(); + HDFSResourceWebRequestValidator validator = new HDFSResourceWebRequestValidator(); + MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo(); + List<FileStatusEntity> result = new ArrayList<>(); List<FileStatus> fileStatuses = null; try { - validator.validate(site, filePath); // First Step would be validating Request - HDFSResourceAccessConfig config = HDFSResourceUtils.getConfig(site); - HDFSFileSystem fileSystem = new HDFSFileSystem(config.getHdfsEndpoint()); + validator.validate(site, filePath); // First Step would be validating Request + Configuration config = repo.getConfig(HDFSResourceConstants.HDFS_APPLICATION, site); + HDFSFileSystem fileSystem = new HDFSFileSystem(config); fileStatuses = fileSystem.browse(filePath); // Join with File Sensitivity Info HDFSResourceSensitivityDataJoiner joiner = new HDFSResourceSensitivityDataJoiner(); - result = joiner.joinFileSensitivity(site, fileStatuses); + result = joiner.joinFileSensitivity(site, fileStatuses); LOG.info("Successfully browsed files in HDFS ."); } catch( Exception ex ) { response.setException(EagleExceptionWrapper.wrap(ex)); @@ -73,4 +74,4 @@ public class HDFSResourceWebResource response.setObj(result); return response; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSFileSystemTest.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSFileSystemTest.java b/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSFileSystemTest.java deleted file mode 100644 index 5943783..0000000 --- a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSFileSystemTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.security.hdfs.test; - -import org.junit.Test; - -import org.apache.eagle.service.security.hdfs.HDFSFileSystem; - -public class HDFSFileSystemTest { - //@Test - public void testHDFSFileSystem() throws Exception { - HDFSFileSystem fileSystem = new HDFSFileSystem("hdfs://sandbox.hortonworks.com:8020"); - System.out.println(fileSystem.browse("/")); - } - - @Test - public void test() { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceAccessConfigTest.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceAccessConfigTest.java b/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceAccessConfigTest.java index 5dfeea6..f046d0a 100644 --- a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceAccessConfigTest.java +++ b/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceAccessConfigTest.java @@ -16,20 +16,30 @@ */ package org.apache.eagle.service.security.hdfs.test; -import org.apache.eagle.service.security.hdfs.HDFSResourceAccessConfig; -import org.apache.eagle.service.security.hdfs.HDFSResourceUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import junit.framework.Assert; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; public class HDFSResourceAccessConfigTest { - //@Test - public void testHDFSResourceAccessConfig() throws Exception { - HDFSResourceAccessConfig config = HDFSResourceUtils.getConfig("cluster1-dc1"); - System.out.println(config); - } - @Test - public void test() { + public void testHDFSResourceAccessConfig() throws Exception { + String configStr = "{\"fs.defaultFS\":\"hdfs://sandbox-nn-ha\",\"dfs.nameservices\":\"sandbox-nn-ha\",\"dfs.ha.namenodes.sandbox-nn-ha\":\"nn1,nn2\",\"dfs.namenode.rpc-address.sandbox-nn-ha.nn1\":\"sandbox-nn.vip.ebay.com:8020\",\"dfs.namenode.rpc-address.sandbox-nn-ha.nn2\":\"sandbox-nn-2.vip.ebay.com:8020\",\"dfs.client.failover.proxy.provider.sandbox-nn-ha\":\"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\",\"hadoop.security.authentication\":\"kerberos\",\"dfs.namenode.kerberos.principal\":\"hadoop/_h...@example.com\"}"; + ObjectMapper mapper = new ObjectMapper(); + Map<String, String> configMap = mapper.readValue(configStr, Map.class); + Map<String, String> result = new HashMap<>(); + result.put("fs.defaultFS" , "hdfs://sandbox-nn-ha"); + result.put("dfs.nameservices", "sandbox-nn-ha"); + result.put("dfs.ha.namenodes.sandbox-nn-ha", "nn1,nn2"); + result.put("dfs.namenode.rpc-address.sandbox-nn-ha.nn1", "sandbox-nn.vip.ebay.com:8020"); + result.put("dfs.namenode.rpc-address.sandbox-nn-ha.nn2", "sandbox-nn-2.vip.ebay.com:8020"); + result.put("dfs.client.failover.proxy.provider.sandbox-nn-ha","org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); + result.put("hadoop.security.authentication", "kerberos"); + result.put("dfs.namenode.kerberos.principal", "hadoop/_h...@example.com"); + Assert.assertEquals(configMap, result); } -} + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceConfigAccessTest.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceConfigAccessTest.java b/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceConfigAccessTest.java deleted file mode 100644 index efd7ac4..0000000 --- a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceConfigAccessTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.security.hdfs.test; - -import org.apache.eagle.service.security.hdfs.HDFSResourceUtils; -import org.junit.Test; - -public class HDFSResourceConfigAccessTest { - - //@Test - public void tesHDFSResourceAccessConfig() throws Exception - { - System.out.println(HDFSResourceUtils.getConfig("cluster1-dc1")); - } - - @Test - public void test() { - - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceResolverTest.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceResolverTest.java b/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceResolverTest.java deleted file mode 100644 index aeb4783..0000000 --- a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceResolverTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.security.hdfs.test; - -import org.apache.eagle.service.security.hdfs.resolver.HDFSResourceResolver; -import org.apache.eagle.service.alert.resolver.AttributeResolveException; -import org.apache.eagle.service.alert.resolver.GenericAttributeResolveRequest; -import org.junit.Test; - -public class HDFSResourceResolverTest { - //@Test - public void testHDFSResourceResolver() throws AttributeResolveException { - GenericAttributeResolveRequest request = new GenericAttributeResolveRequest("/user","cluster1-dc1"); - HDFSResourceResolver resolve = new HDFSResourceResolver(); - System.out.println(resolve.resolve(request)); - } - - @Test - public void test() { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceSensitivityServiceTest.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceSensitivityServiceTest.java b/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceSensitivityServiceTest.java deleted file mode 100644 index 983ca0c..0000000 --- a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceSensitivityServiceTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.security.hdfs.test; - -import org.apache.eagle.service.security.hdfs.HDFSResourceSensitivityService; -import org.junit.Test; - -public class HDFSResourceSensitivityServiceTest { - - //@Test - public void testgetAllFileSensitivityService() { - HDFSResourceSensitivityService service = new HDFSResourceSensitivityService(); - service.getAllFileSensitivityMap(); - - } - - //@Test - public void testgetFileSensitivityBySite() { - HDFSResourceSensitivityService service = new HDFSResourceSensitivityService(); - service.getFileSensitivityMapBySite("cluster1-dc1"); - - } - - @Test - public void test() { - - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceWebTest.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceWebTest.java b/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceWebTest.java deleted file mode 100644 index c189cfc..0000000 --- a/eagle-security/eagle-security-hdfs-web/src/test/java/org/apache/eagle/service/security/hdfs/test/HDFSResourceWebTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.security.hdfs.test; - -import org.apache.eagle.service.security.hdfs.rest.HDFSResourceWebResource; -import org.junit.Test; - -public class HDFSResourceWebTest { - - //@Test - public void testWebHDFSResourceAPI() { - HDFSResourceWebResource resource = new HDFSResourceWebResource(); - System.out.println(resource.getHDFSResource("cluster1-dc1", "/user")); - - } - - @Test - public void test() { - - } -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-webservice/src/main/java/org/apache/eagle/service/security/auth/AuthoritiesPopulator.java ---------------------------------------------------------------------- diff --git a/eagle-webservice/src/main/java/org/apache/eagle/service/security/auth/AuthoritiesPopulator.java b/eagle-webservice/src/main/java/org/apache/eagle/service/security/auth/AuthoritiesPopulator.java new file mode 100644 index 0000000..95eb047 --- /dev/null +++ b/eagle-webservice/src/main/java/org/apache/eagle/service/security/auth/AuthoritiesPopulator.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.service.security.auth; + + +import org.springframework.ldap.core.ContextSource; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator; + +import java.util.HashSet; +import java.util.Set; + +public class AuthoritiesPopulator extends DefaultLdapAuthoritiesPopulator { + + String adminRole; + SimpleGrantedAuthority adminRoleAsAuthority; + + SimpleGrantedAuthority adminAuthority = new SimpleGrantedAuthority("ROLE_ADMIN"); + SimpleGrantedAuthority defaultAuthority = new SimpleGrantedAuthority("ROLE_USER"); + + /** + * @param contextSource + * @param groupSearchBase + */ + public AuthoritiesPopulator(ContextSource contextSource, String groupSearchBase, String adminRole, String defaultRole) { + super(contextSource, groupSearchBase); + this.adminRole = adminRole; + this.adminRoleAsAuthority = new SimpleGrantedAuthority(adminRole); + } + + @Override + public Set<GrantedAuthority> getGroupMembershipRoles(String userDn, String username) { + Set<GrantedAuthority> authorities = super.getGroupMembershipRoles(userDn, username); + Set<GrantedAuthority> newAuthorities = new HashSet<>(); + + if (authorities.contains(adminRoleAsAuthority)) { + newAuthorities.add(adminAuthority); + } else { + newAuthorities.add(defaultAuthority); + } + + return newAuthorities; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/d394a8e3/eagle-webservice/src/main/resources/eagleSecurity.xml ---------------------------------------------------------------------- diff --git a/eagle-webservice/src/main/resources/eagleSecurity.xml b/eagle-webservice/src/main/resources/eagleSecurity.xml index b8683dc..952f2e7 100644 --- a/eagle-webservice/src/main/resources/eagleSecurity.xml +++ b/eagle-webservice/src/main/resources/eagleSecurity.xml @@ -23,13 +23,11 @@ http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.1.xsd";> - <scr:http auto-config="true" use-expressions="true"> + <scr:http auto-config="true" use-expressions="true"> <!-- Support HTTP Basic Auth--> <scr:http-basic entry-point-ref="unauthorisedEntryPoint"/> <scr:intercept-url pattern="/rest/entities" method="POST" access="hasRole('ROLE_ADMIN')" /> <scr:intercept-url pattern="/rest/entities/delete" method="POST" access="hasRole('ROLE_ADMIN')" /> - <scr:intercept-url pattern="/rest/module/*" method="DELETE" access="hasRole('ROLE_ADMIN')" /> - <scr:intercept-url pattern="/rest/module/*" method="POST" access="hasRole('ROLE_ADMIN')" /> <scr:intercept-url pattern="/rest/list" method="POST" access="hasRole('ROLE_ADMIN')" /> <scr:intercept-url pattern="/rest/status" method="GET" access="permitAll" /> <scr:intercept-url pattern="/rest/*" access="isAuthenticated()" /> @@ -41,49 +39,48 @@ <bean id="logoutSuccessHandler" class="org.apache.eagle.service.security.auth.LogoutSuccessHandlerImpl" /> <bean id="unauthorisedEntryPoint" class="org.springframework.security.web.authentication.Http403ForbiddenEntryPoint" /> - <beans profile="default"> - <bean id="ldapUserAuthProvider" class="org.springframework.security.ldap.authentication.LdapAuthenticationProvider"> - <constructor-arg> - <bean class="org.springframework.security.ldap.authentication.BindAuthenticator"> - <constructor-arg ref="ldapSource" /> - <property name="userSearch"> - <bean id="userSearch" class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch"> - <constructor-arg index="0" value="${ldap.user.searchBase}" /> - <constructor-arg index="1" value="${ldap.user.searchPattern}" /> - <constructor-arg index="2" ref="ldapSource" /> - </bean> - </property> - </bean> - </constructor-arg> - <constructor-arg> - <bean class="org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator"> - <constructor-arg index="0" ref="ldapSource" /> - <constructor-arg index="1" value="${ldap.user.groupSearchBase}" /> - <property name="groupSearchFilter" value="uniqueMember={0}"/> - <property name="convertToUpperCase" value="true" /> - <property name="rolePrefix" value="ROLE_" /> - </bean> - </constructor-arg> - </bean> + <beans profile="default"> + <bean id="ldapUserAuthProvider" class="org.springframework.security.ldap.authentication.LdapAuthenticationProvider"> + <constructor-arg> + <bean class="org.springframework.security.ldap.authentication.BindAuthenticator"> + <constructor-arg ref="ldapSource" /> + <property name="userSearch"> + <bean id="userSearch" class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch"> + <constructor-arg index="0" value="${ldap.user.searchBase}" /> + <constructor-arg index="1" value="${ldap.user.searchPattern}" /> + <constructor-arg index="2" ref="ldapSource" /> + </bean> + </property> + </bean> + </constructor-arg> + <constructor-arg> + <bean class="org.apache.eagle.service.security.auth.AuthoritiesPopulator"> + <constructor-arg index="0" ref="ldapSource" /> + <constructor-arg index="1" value="${ldap.user.groupSearchBase}" /> + <constructor-arg index="2" value="${acl.adminRole}" /> + <constructor-arg index="3" value="${acl.defaultRole}" /> + </bean> + </constructor-arg> + </bean> - <scr:authentication-manager alias="authenticationManager"> - <!-- do user ldap auth --> - <scr:authentication-provider ref="ldapUserAuthProvider"></scr:authentication-provider> - </scr:authentication-manager> + <scr:authentication-manager alias="authenticationManager"> + <!-- do user ldap auth --> + <scr:authentication-provider ref="ldapUserAuthProvider"></scr:authentication-provider> + </scr:authentication-manager> - <bean id="ldapSource" class="org.springframework.security.ldap.DefaultSpringSecurityContextSource"> - <constructor-arg value="${ldap.server}" /> - <property name="userDn" value="${ldap.username}" /> - <property name="password" value="${ldap.password}" /> - </bean> - </beans> + <bean id="ldapSource" class="org.springframework.security.ldap.DefaultSpringSecurityContextSource"> + <constructor-arg value="${ldap.server}" /> + <property name="userDn" value="${ldap.username}" /> + <property name="password" value="${ldap.password}" /> + </bean> + </beans> <beans profile="sandbox,testing"> <scr:authentication-manager alias="authenticationManager"> <scr:authentication-provider> <scr:user-service> <!-- user admin has role ADMIN, user eagle has role USER, both have password "secret" --> - <scr:user name="eagle" password="$2a$10$TwALMRHpSetDaeTurg9rj.DnIdOde4fkQGBSPG3fVqtH.G5ZH8sQK" authorities="ROLE_USER" /> + <scr:user name="eagle" password="$2a$10$TwALMRHpSetDaeTurg9rj.DnIdOde4fkQGBSPG3fVqtH.G5ZH8sQK" authorities="ROLE_USER" /> <scr:user name="admin" password="$2a$10$TwALMRHpSetDaeTurg9rj.DnIdOde4fkQGBSPG3fVqtH.G5ZH8sQK" authorities="ROLE_ADMIN" /> </scr:user-service> <scr:password-encoder ref="passwordEncoder" />
