Repository: nifi Updated Branches: refs/heads/master d28b1172d -> 2201f7746
NIFI-5714 - Hive[3]ConnectionPool - Kerberos Authentication issue/misleading add @Ignore on unit test... Signed-off-by: Matthew Burgess <[email protected]> This closes #3086 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2201f774 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2201f774 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2201f774 Branch: refs/heads/master Commit: 2201f7746fd16874aefbd12d546565f5d105ab04 Parents: d28b117 Author: Pierre Villard <[email protected]> Authored: Wed Oct 17 01:04:16 2018 +0200 Committer: Matthew Burgess <[email protected]> Committed: Wed Oct 24 13:33:30 2018 -0400 ---------------------------------------------------------------------- .../nifi/dbcp/hive/HiveConnectionPool.java | 2 + .../nifi/dbcp/hive/HiveConnectionPoolTest.java | 84 +++++++++++++++++--- .../src/test/resources/hive-site-security.xml | 4 + .../src/test/resources/krb5.conf | 10 +++ .../nifi/dbcp/hive/Hive3ConnectionPool.java | 2 + .../nifi/dbcp/hive/Hive3ConnectionPoolTest.java | 84 +++++++++++++++++--- .../src/test/resources/hive-site-security.xml | 4 + .../src/test/resources/krb5.conf | 10 +++ 8 files changed, 174 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java index 2e40254..378799e 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java @@ -306,10 +306,12 @@ public class HiveConnectionPool extends AbstractControllerService implements Hiv } log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); + try { ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab); } catch (AuthenticationFailedException ae) { log.error(ae.getMessage(), ae); + throw new InitializationException(ae); } getLogger().info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java index 96dfb4f..95873b2 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java @@ -17,42 +17,56 @@ package org.apache.nifi.dbcp.hive; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedExceptionAction; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + import org.apache.commons.dbcp.BasicDataSource; import org.apache.hadoop.security.UserGroupInformation; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.hadoop.KerberosProperties; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.registry.VariableDescriptor; +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockConfigurationContext; import org.apache.nifi.util.MockVariableRegistry; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.UndeclaredThrowableException; -import java.security.PrivilegedExceptionAction; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class HiveConnectionPoolTest { private UserGroupInformation userGroupInformation; private HiveConnectionPool hiveConnectionPool; private BasicDataSource basicDataSource; private ComponentLog componentLog; + private KerberosProperties kerberosProperties; + private File krb5conf = new File("src/test/resources/krb5.conf"); @Before public void setup() throws Exception { + // have to initialize this system property before anything else + System.setProperty("java.security.krb5.conf", krb5conf.getAbsolutePath()); + System.setProperty("java.security.krb5.realm", "nifi.com"); + System.setProperty("java.security.krb5.kdc", "nifi.kdc"); + userGroupInformation = mock(UserGroupInformation.class); basicDataSource = mock(BasicDataSource.class); componentLog = mock(ComponentLog.class); + kerberosProperties = mock(KerberosProperties.class); when(userGroupInformation.doAs(isA(PrivilegedExceptionAction.class))).thenAnswer(invocation -> { try { @@ -63,6 +77,19 @@ public class HiveConnectionPoolTest { throw new UndeclaredThrowableException(e); } }); + + when(kerberosProperties.getKerberosKeytab()).thenReturn(new PropertyDescriptor.Builder() + .name("Kerberos Principal") + .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build()); + + when(kerberosProperties.getKerberosPrincipal()).thenReturn(new PropertyDescriptor.Builder() + .name("Kerberos Keytab") + .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build()); + initPool(); } @@ -80,6 +107,10 @@ public class HiveConnectionPoolTest { Field componentLogField = AbstractControllerService.class.getDeclaredField("logger"); componentLogField.setAccessible(true); componentLogField.set(hiveConnectionPool, componentLog); + + Field kerberosPropertiesField = HiveConnectionPool.class.getDeclaredField("kerberosProperties"); + kerberosPropertiesField.setAccessible(true); + kerberosPropertiesField.set(hiveConnectionPool, kerberosProperties); } @Test(expected = ProcessException.class) @@ -135,4 +166,31 @@ public class HiveConnectionPoolTest { assertEquals(10000L, basicDataSource.getMaxWait()); assertEquals(URL, hiveConnectionPool.getConnectionURL()); } + + @Ignore("Kerberos does not seem to be properly handled in Travis build, but, locally, this test should successfully run") + @Test(expected = InitializationException.class) + public void testKerberosAuthException() throws Exception { + final String URL = "jdbc:hive2://localhost:10000/default"; + final String conf = "src/test/resources/hive-site-security.xml"; + final String ktab = "src/test/resources/fake.keytab"; + final String kprinc = "[email protected]"; + + KerberosProperties kerbProperties = new KerberosProperties(krb5conf); + + Map<PropertyDescriptor, String> props = new HashMap<PropertyDescriptor, String>() {{ + put(HiveConnectionPool.DATABASE_URL, "${url}"); + put(HiveConnectionPool.HIVE_CONFIGURATION_RESOURCES, "${conf}"); + put(kerbProperties.getKerberosKeytab(), "${ktab}"); + put(kerbProperties.getKerberosPrincipal(), "${kprinc}"); + }}; + + MockVariableRegistry registry = new MockVariableRegistry(); + registry.setVariable(new VariableDescriptor("url"), URL); + registry.setVariable(new VariableDescriptor("conf"), conf); + registry.setVariable(new VariableDescriptor("ktab"), ktab); + registry.setVariable(new VariableDescriptor("kprinc"), kprinc); + + MockConfigurationContext context = new MockConfigurationContext(props, null, registry); + hiveConnectionPool.onConfigured(context); + } } http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml index 07fd74c..4d64c95 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/hive-site-security.xml @@ -23,4 +23,8 @@ <name>hive.server2.authentication</name> <value>KERBEROS</value> </property> + <property> + <name>hadoop.security.authentication</name> + <value>kerberos</value> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf index e69de29..323da39 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/resources/krb5.conf @@ -0,0 +1,10 @@ +[libdefaults] + default_realm = EXAMPLE.COM + dns_lookup_kdc = false + dns_lookup_realm = false + +[realms] + EXAMPLE.COM = { + kdc = kerberos.example.com + admin_server = kerberos.example.com + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java index b0662b8..c2042bb 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java @@ -304,10 +304,12 @@ public class Hive3ConnectionPool extends AbstractControllerService implements Hi } log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); + try { ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab); } catch (AuthenticationFailedException ae) { log.error(ae.getMessage(), ae); + throw new InitializationException(ae); } getLogger().info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java index 5d9f87c..b38f41e 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPoolTest.java @@ -17,42 +17,56 @@ package org.apache.nifi.dbcp.hive; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedExceptionAction; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + import org.apache.commons.dbcp.BasicDataSource; import org.apache.hadoop.security.UserGroupInformation; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.hadoop.KerberosProperties; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.registry.VariableDescriptor; +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockConfigurationContext; import org.apache.nifi.util.MockVariableRegistry; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.UndeclaredThrowableException; -import java.security.PrivilegedExceptionAction; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class Hive3ConnectionPoolTest { private UserGroupInformation userGroupInformation; private Hive3ConnectionPool hive3ConnectionPool; private BasicDataSource basicDataSource; private ComponentLog componentLog; + private KerberosProperties kerberosProperties; + private File krb5conf = new File("src/test/resources/krb5.conf"); @Before public void setup() throws Exception { + // have to initialize this system property before anything else + System.setProperty("java.security.krb5.conf", krb5conf.getAbsolutePath()); + System.setProperty("java.security.krb5.realm", "nifi.com"); + System.setProperty("java.security.krb5.kdc", "nifi.kdc"); + userGroupInformation = mock(UserGroupInformation.class); basicDataSource = mock(BasicDataSource.class); componentLog = mock(ComponentLog.class); + kerberosProperties = mock(KerberosProperties.class); when(userGroupInformation.doAs(isA(PrivilegedExceptionAction.class))).thenAnswer(invocation -> { try { @@ -63,6 +77,19 @@ public class Hive3ConnectionPoolTest { throw new UndeclaredThrowableException(e); } }); + + when(kerberosProperties.getKerberosKeytab()).thenReturn(new PropertyDescriptor.Builder() + .name("Kerberos Principal") + .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build()); + + when(kerberosProperties.getKerberosPrincipal()).thenReturn(new PropertyDescriptor.Builder() + .name("Kerberos Keytab") + .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build()); + initPool(); } @@ -80,6 +107,10 @@ public class Hive3ConnectionPoolTest { Field componentLogField = AbstractControllerService.class.getDeclaredField("logger"); componentLogField.setAccessible(true); componentLogField.set(hive3ConnectionPool, componentLog); + + Field kerberosPropertiesField = Hive3ConnectionPool.class.getDeclaredField("kerberosProperties"); + kerberosPropertiesField.setAccessible(true); + kerberosPropertiesField.set(hive3ConnectionPool, kerberosProperties); } @Test(expected = ProcessException.class) @@ -135,4 +166,31 @@ public class Hive3ConnectionPoolTest { assertEquals(10000L, basicDataSource.getMaxWait()); assertEquals(URL, hive3ConnectionPool.getConnectionURL()); } + + @Ignore("Kerberos does not seem to be properly handled in Travis build, but, locally, this test should successfully run") + @Test(expected = InitializationException.class) + public void testKerberosAuthException() throws Exception { + final String URL = "jdbc:hive2://localhost:10000/default"; + final String conf = "src/test/resources/hive-site-security.xml"; + final String ktab = "src/test/resources/fake.keytab"; + final String kprinc = "[email protected]"; + + KerberosProperties kerbProperties = new KerberosProperties(krb5conf); + + Map<PropertyDescriptor, String> props = new HashMap<PropertyDescriptor, String>() {{ + put(Hive3ConnectionPool.DATABASE_URL, "${url}"); + put(Hive3ConnectionPool.HIVE_CONFIGURATION_RESOURCES, "${conf}"); + put(kerbProperties.getKerberosKeytab(), "${ktab}"); + put(kerbProperties.getKerberosPrincipal(), "${kprinc}"); + }}; + + MockVariableRegistry registry = new MockVariableRegistry(); + registry.setVariable(new VariableDescriptor("url"), URL); + registry.setVariable(new VariableDescriptor("conf"), conf); + registry.setVariable(new VariableDescriptor("ktab"), ktab); + registry.setVariable(new VariableDescriptor("kprinc"), kprinc); + + MockConfigurationContext context = new MockConfigurationContext(props, null, registry); + hive3ConnectionPool.onConfigured(context); + } } http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml index 07fd74c..4d64c95 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/hive-site-security.xml @@ -23,4 +23,8 @@ <name>hive.server2.authentication</name> <value>KERBEROS</value> </property> + <property> + <name>hadoop.security.authentication</name> + <value>kerberos</value> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/2201f774/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf index e69de29..323da39 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/resources/krb5.conf @@ -0,0 +1,10 @@ +[libdefaults] + default_realm = EXAMPLE.COM + dns_lookup_kdc = false + dns_lookup_realm = false + +[realms] + EXAMPLE.COM = { + kdc = kerberos.example.com + admin_server = kerberos.example.com + } \ No newline at end of file
