Repository: apex-core Updated Branches: refs/heads/master 0670eb3e0 -> bfc1eb874
APEXCORE-517 Added BASIC authentication support to be able to authenticate with Hadoop web services with BASIC authentication enabled Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/bfc1eb87 Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/bfc1eb87 Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/bfc1eb87 Branch: refs/heads/master Commit: bfc1eb87436a324c1fdee130cb5c66c839fc76a2 Parents: 0670eb3 Author: Pramod Immaneni <[email protected]> Authored: Mon Sep 5 13:59:13 2016 -0700 Committer: Pramod Immaneni <[email protected]> Committed: Wed Sep 14 07:29:42 2016 -0700 ---------------------------------------------------------------------- .../java/com/datatorrent/stram/cli/ApexCli.java | 2 + .../datatorrent/stram/security/AuthScheme.java | 50 ++++++++++++++ .../datatorrent/stram/util/SecurityUtils.java | 25 ++++++- .../stram/util/WebServicesClient.java | 71 +++++++++++++++----- .../stram/util/SecurityUtilsTest.java | 23 ++++++- .../stram/util/WebServicesClientTest.java | 32 +++++++++ .../test/resources/security/dt-site-basic.xml | 33 +++++++++ .../resources/security/dt-site-kerberos.xml | 33 +++++++++ engine/src/test/resources/security/user.keytab | 1 + pom.xml | 2 + 10 files changed, 252 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java index 9db488f..235ba58 100644 --- a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java +++ b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java @@ -123,6 +123,7 @@ import com.datatorrent.stram.plan.logical.requests.SetPortAttributeRequest; import com.datatorrent.stram.plan.logical.requests.SetStreamAttributeRequest; import com.datatorrent.stram.security.StramUserLogin; import com.datatorrent.stram.util.JSONSerializationProvider; +import com.datatorrent.stram.util.SecurityUtils; import com.datatorrent.stram.util.VersionInfo; import com.datatorrent.stram.util.WebServicesClient; import com.datatorrent.stram.webapp.OperatorDiscoverer; @@ -1131,6 +1132,7 @@ public class ApexCli public void init() throws IOException { conf = StramClientUtils.addDTSiteResources(new YarnConfiguration()); + SecurityUtils.init(conf); fs = StramClientUtils.newFileSystemInstance(conf); stramAgent = new StramAgent(fs, conf); http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java b/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java new file mode 100644 index 0000000..f39dddf --- /dev/null +++ b/engine/src/main/java/com/datatorrent/stram/security/AuthScheme.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datatorrent.stram.security; + +/** + * Authentication scheme + * + * BASIC: BASIC HTTP authentication - RFC 2617 + * SPNEGO: Kerberos based SPNEGO authentication - RFC 4559 + * KERBEROS: Kerberos authentication - RFC 4120 + */ +public enum AuthScheme +{ + BASIC("basic"), SPNEGO("kerberos"), KERBEROS("kerberos-standard"); + + String name; + + AuthScheme(String name) + { + this.name = name; + } + + public String getName() + { + return name; + } + + @Override + public String toString() + { + return name; + } + +} http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java b/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java index 0d54511..349d09a 100644 --- a/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java +++ b/engine/src/main/java/com/datatorrent/stram/util/SecurityUtils.java @@ -23,6 +23,8 @@ import org.apache.hadoop.security.UserGroupInformation; import com.datatorrent.api.Context; import com.datatorrent.api.Context.StramHTTPAuthentication; +import com.datatorrent.stram.security.AuthScheme; +import com.datatorrent.stram.security.StramUserLogin; /** * @@ -43,15 +45,20 @@ public class SecurityUtils hadoopWebSecurityEnabled = stramWebSecurityEnabled = UserGroupInformation.isSecurityEnabled(); } + public static void init(Configuration configuration) + { + init(configuration, null); + } + public static void init(Configuration configuration, StramHTTPAuthentication stramHTTPAuth) { hadoopWebSecurityEnabled = false; String authValue = configuration.get(HADOOP_HTTP_AUTH_PROP); if ((authValue != null) && !authValue.equals(HADOOP_HTTP_AUTH_VALUE_SIMPLE)) { hadoopWebSecurityEnabled = true; + initAuth(configuration); } // Stram http auth may not be specified and is null but still set a default - boolean authDefault = false; if (stramHTTPAuth != null) { if (stramHTTPAuth == Context.StramHTTPAuthentication.FOLLOW_HADOOP_HTTP_AUTH) { stramWebSecurityEnabled = hadoopWebSecurityEnabled; @@ -65,6 +72,22 @@ public class SecurityUtils } } + private static void initAuth(final Configuration configuration) + { + // Authentication scheme is not unambiguously known because in Hadoop authentication, authentication type can be + // specified as an implementation class, furthermore authentication types like SASL wrap other mechanisms like BASIC + // or SPNEGO underneath and the wrapped scheme is not known till the authentication negotiation process + WebServicesClient.initAuth(new WebServicesClient.ConfigProvider() + { + @Override + public String getProperty(AuthScheme scheme, String name) + { + StringBuilder propNamesb = new StringBuilder(StramUserLogin.DT_AUTH_PREFIX).append(scheme.getName()).append(".").append(name); + return configuration.get(propNamesb.toString()); + } + }); + } + public static boolean isHadoopWebSecurityEnabled() { return hadoopWebSecurityEnabled; http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java b/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java index 1712080..73d5532 100644 --- a/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java +++ b/engine/src/main/java/com/datatorrent/stram/util/WebServicesClient.java @@ -30,10 +30,13 @@ import org.slf4j.LoggerFactory; import org.apache.http.auth.AuthSchemeProvider; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.AuthSchemes; import org.apache.http.config.Lookup; import org.apache.http.config.RegistryBuilder; +import org.apache.http.impl.auth.BasicSchemeFactory; +import org.apache.http.impl.auth.KerberosSchemeFactory; import org.apache.http.impl.auth.SPNegoSchemeFactory; import org.apache.http.impl.client.BasicCookieStore; import org.apache.http.impl.client.BasicCredentialsProvider; @@ -49,6 +52,8 @@ import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.ClientFilter; import com.sun.jersey.client.apache4.ApacheHttpClient4Handler; +import com.datatorrent.stram.security.AuthScheme; + /** * <p>WebServicesClient class.</p> * @@ -61,6 +66,8 @@ public class WebServicesClient private static final PoolingHttpClientConnectionManager connectionManager; private static final CredentialsProvider credentialsProvider; + private static final RegistryBuilder<AuthSchemeProvider> registryBuilder; + private static Lookup<AuthSchemeProvider> authRegistry; private static final int DEFAULT_CONNECT_TIMEOUT = 10000; private static final int DEFAULT_READ_TIMEOUT = 10000; @@ -68,27 +75,55 @@ public class WebServicesClient private final Set<ClientFilter> clientFilters = new HashSet<>(); + private static final Credentials DEFAULT_TOKEN_CREDENTIALS = new Credentials() + { + @Override + public Principal getUserPrincipal() + { + return null; + } + + @Override + public String getPassword() + { + return null; + } + }; + static { connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setMaxTotal(200); connectionManager.setDefaultMaxPerRoute(5); + registryBuilder = RegistryBuilder.<AuthSchemeProvider>create(); credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new Credentials() - { + // By default add SPNEGO so that it works even if auth is not explictly configured like before, in future + // move it to auth setup below + registryBuilder.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)); + credentialsProvider.setCredentials(AuthScope.ANY, DEFAULT_TOKEN_CREDENTIALS); + authRegistry = registryBuilder.build(); + } - @Override - public Principal getUserPrincipal() - { - return null; - } + public static void initAuth(ConfigProvider configuration) + { + // Adding BASIC auth + AuthScheme scheme = AuthScheme.BASIC; + String username = configuration.getProperty(scheme, "username"); + String password = configuration.getProperty(scheme, "password"); + if ((username != null) && (password != null)) { + LOG.info("Setting up scheme {}", scheme); + registryBuilder.register(AuthSchemes.BASIC, new BasicSchemeFactory()); + AuthScope authScope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.BASIC); + Credentials credentials = new UsernamePasswordCredentials(username, password); + credentialsProvider.setCredentials(authScope, credentials); + } else if ((username != null) || (password != null)) { + LOG.warn("Not setting up scheme {}, missing credentials {}", scheme, (username == null) ? "username" : "password"); + } - @Override - public String getPassword() - { - return null; - } + // Adding kerberos standard auth + registryBuilder.register(AuthSchemes.KERBEROS, new KerberosSchemeFactory()); + credentialsProvider.setCredentials(AuthScope.ANY, DEFAULT_TOKEN_CREDENTIALS); - }); + authRegistry = registryBuilder.build(); } public WebServicesClient() @@ -105,10 +140,7 @@ public class WebServicesClient HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); httpClientBuilder.setConnectionManager(connectionManager); httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - Lookup<AuthSchemeProvider> authProviders = RegistryBuilder.<AuthSchemeProvider>create() - .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)) - .build(); - httpClientBuilder.setDefaultAuthSchemeRegistry(authProviders); + httpClientBuilder.setDefaultAuthSchemeRegistry(authRegistry); ApacheHttpClient4Handler httpClientHandler = new ApacheHttpClient4Handler(httpClientBuilder.build(), new BasicCookieStore(), false); client = new Client(httpClientHandler, config); } else { @@ -183,6 +215,11 @@ public class WebServicesClient }); } + public interface ConfigProvider + { + String getProperty(AuthScheme scheme, String name); + } + /** * * @param <T> http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java b/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java index 86d4eae..a2623fe 100644 --- a/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java +++ b/engine/src/test/java/com/datatorrent/stram/util/SecurityUtilsTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import org.apache.hadoop.conf.Configuration; import com.datatorrent.api.Context; +import com.datatorrent.stram.security.AuthScheme; /** * @@ -34,12 +35,30 @@ public class SecurityUtilsTest public void testStramWebSecurity() { checkWebSecurity(false, false); - Configuration conf = new Configuration(); + Configuration conf = setupConfiguration(null); checkSecurityConfiguration(conf, new boolean[][]{{false, false}, {false, true}, {false, false}, {false, false}, {false, false}}); - conf.set(SecurityUtils.HADOOP_HTTP_AUTH_PROP, "kerberos"); + conf = setupConfiguration(AuthScheme.SPNEGO); checkSecurityConfiguration(conf, new boolean[][]{{true, false}, {true, true}, {true, false}, {true, false}, {true, true}}); } + @Test + public void testInitAuth() throws NoSuchFieldException, IllegalAccessException + { + Configuration conf = setupConfiguration(AuthScheme.BASIC); + SecurityUtils.init(conf); + WebServicesClientTest.checkUserCredentials("testuser", "testpass"); + } + + private Configuration setupConfiguration(AuthScheme authScheme) + { + Configuration conf = new Configuration(); + if (authScheme != null) { + conf.set(SecurityUtils.HADOOP_HTTP_AUTH_PROP, authScheme.getName()); + conf.addResource("security/dt-site-" + authScheme.getName() + ".xml"); + } + return conf; + } + private void checkSecurityConfiguration(Configuration conf, boolean[][] securityConf) { Assert.assertEquals("Number variations", 5, securityConf.length); http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java b/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java index f07a7cf..fd422f0 100644 --- a/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java +++ b/engine/src/test/java/com/datatorrent/stram/util/WebServicesClientTest.java @@ -18,15 +18,24 @@ */ package com.datatorrent.stram.util; +import java.lang.reflect.Field; + import org.junit.Assert; import org.junit.Test; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.AuthSchemes; /** * */ public class WebServicesClientTest { + private static final String CREDENTIALS_PROVIDER_FIELD = "credentialsProvider"; + @Test public void testFilterPresent() { @@ -35,4 +44,27 @@ public class WebServicesClientTest webServicesClient.addFilter(clientFilter); Assert.assertTrue("Filter present", webServicesClient.isFilterPresent(clientFilter)); } + + public static void checkUserCredentials(String username, String password) throws NoSuchFieldException, + IllegalAccessException + { + CredentialsProvider provider = getCredentialsProvider(); + AuthScope authScope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.BASIC); + Credentials credentials = provider.getCredentials(authScope); + Assert.assertNotNull("Credentials", credentials); + Assert.assertTrue("Credentials type is user", UsernamePasswordCredentials.class.isAssignableFrom(credentials.getClass())); + UsernamePasswordCredentials pwdCredentials = (UsernamePasswordCredentials)credentials; + Assert.assertEquals("Username", username, pwdCredentials.getUserName()); + Assert.assertEquals("Password", password, pwdCredentials.getPassword()); + } + + private static CredentialsProvider getCredentialsProvider() throws NoSuchFieldException, IllegalAccessException + { + Field field = WebServicesClient.class.getDeclaredField(CREDENTIALS_PROVIDER_FIELD); + field.setAccessible(true); + CredentialsProvider credentials = (CredentialsProvider)field.get(null); + field.setAccessible(false); + return credentials; + } + } http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/resources/security/dt-site-basic.xml ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/security/dt-site-basic.xml b/engine/src/test/resources/security/dt-site-basic.xml new file mode 100644 index 0000000..e46e6a9 --- /dev/null +++ b/engine/src/test/resources/security/dt-site-basic.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +<configuration> + <property> + <name>dt.authentication.basic.username</name> + <value>testuser</value> + </property> + <property> + <name>dt.authentication.basic.password</name> + <value>testpass</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/resources/security/dt-site-kerberos.xml ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/security/dt-site-kerberos.xml b/engine/src/test/resources/security/dt-site-kerberos.xml new file mode 100644 index 0000000..ab9ee1c --- /dev/null +++ b/engine/src/test/resources/security/dt-site-kerberos.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +<configuration> + <property> + <name>dt.authentication.kerberos.principal</name> + <value>user/group@domain</value> + </property> + <property> + <name>dt.authentication.keberos.keytab</name> + <value>/security/user-keytab</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/engine/src/test/resources/security/user.keytab ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/security/user.keytab b/engine/src/test/resources/security/user.keytab new file mode 100644 index 0000000..f89b777 --- /dev/null +++ b/engine/src/test/resources/security/user.keytab @@ -0,0 +1 @@ +Fake keytab http://git-wip-us.apache.org/repos/asf/apex-core/blob/bfc1eb87/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e48adc3..6107df7 100644 --- a/pom.xml +++ b/pom.xml @@ -116,6 +116,7 @@ <exclude>**/*.md</exclude> <exclude>**/*.txt</exclude> <exclude>**/*.yml</exclude> + <exclude>**/src/test/resources/**/*.keytab</exclude> <exclude>**/*.importorder</exclude> <exclude>**/archetype-resources/**</exclude> <exclude>misc/ide-templates/**</exclude> @@ -192,6 +193,7 @@ <exclude>.idea/**</exclude> <exclude>**/src/test/resources/**/MANIFEST.MF</exclude> <exclude>**/src/test/resources/**/*.json</exclude> + <exclude>**/src/test/resources/**/*.keytab</exclude> <exclude>**/resources/META-INF/services/**</exclude> <exclude>**/archetype-resources/**</exclude> <exclude>**/*.md</exclude>
