Repository: sqoop Updated Branches: refs/heads/trunk 1378520e4 -> 184452908
SQOOP-3222: Test HBase kerberized connectivity (Szabolcs Vasas via Egyed Boglarka) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/18445290 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/18445290 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/18445290 Branch: refs/heads/trunk Commit: 18445290810b1df035e06fb074064d6b9c1d6e90 Parents: 1378520 Author: Boglarka Egyed <[email protected]> Authored: Wed Aug 30 11:28:31 2017 +0200 Committer: Boglarka Egyed <[email protected]> Committed: Wed Aug 30 11:28:31 2017 +0200 ---------------------------------------------------------------------- build.xml | 11 -- ivy.xml | 2 + .../hbase/HBaseKerberizedConnectivityTest.java | 33 ++++ .../com/cloudera/sqoop/hbase/HBaseTestCase.java | 163 +++++++++---------- .../kerberos/KerberosConfigurationProvider.java | 29 ++++ .../kerberos/MiniKdcInfrastructure.java | 27 +++ .../kerberos/MiniKdcInfrastructureRule.java | 122 ++++++++++++++ 7 files changed, 289 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 5f02dcf..5836a75 100644 --- a/build.xml +++ b/build.xml @@ -230,11 +230,6 @@ <property name="sqoop.test.sqlserver.database" value="sqooptest"/> - <property name="java.security.krb5.realm" - value="OX.AC.UK"/> - - <property name="java.security.krb5.kdc" - value="kdc0.ox.ac.uk:kdc1.ox.ac.uk"/> <property name="ms.sqlserver.username" value="SQOOPUSER"/> @@ -865,12 +860,6 @@ <sysproperty key="sqoop.test.msserver.connector.factory" value="${sqoop.test.msserver.connector.factory}"/> - <sysproperty key="java.security.krb5.realm" - value="${java.security.krb5.realm}"/> - - <sysproperty key="java.security.krb5.kdc" - value="${java.security.krb5.kdc}"/> - <sysproperty key="sqoop.test.db2.connectstring.host_url" value="${sqoop.test.db2.connectstring.host_url}" /> <sysproperty key="sqoop.test.db2.connectstring.database" value="${sqoop.test.db2.connectstring.database}" /> <sysproperty key="sqoop.test.db2.connectstring.username" value="${sqoop.test.db2.connectstring.username}" /> http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/ivy.xml ---------------------------------------------------------------------- diff --git a/ivy.xml b/ivy.xml index e4b45bf..601aa01 100644 --- a/ivy.xml +++ b/ivy.xml @@ -81,6 +81,8 @@ under the License. <dependency org="org.aspectj" name="aspectjrt" rev="${aspectj.version}" conf="common->default"/> + <dependency org="org.apache.hadoop" name="hadoop-minikdc" rev="${hadoop.version}" conf="test->default" /> + <!-- Common dependencies for Sqoop --> <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}" conf="common->default"/> http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java b/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java new file mode 100644 index 0000000..3c027ad --- /dev/null +++ b/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java @@ -0,0 +1,33 @@ +package com.cloudera.sqoop.hbase; + +import org.apache.sqoop.infrastructure.kerberos.MiniKdcInfrastructureRule; +import org.junit.ClassRule; +import org.junit.Test; + +import java.io.IOException; + +public class HBaseKerberizedConnectivityTest extends HBaseTestCase { + + private static final String HBASE_TABLE_NAME = "KerberosTest"; + private static final String HBASE_COLUMN_FAMILY = "TestColumnFamily"; + private static final String TEST_ROW_KEY = "0"; + private static final String TEST_ROW_VALUE = "1"; + private static final String[] COLUMN_TYPES = { "INT", "INT" }; + + @ClassRule + public static MiniKdcInfrastructureRule miniKdcInfrastructure = new MiniKdcInfrastructureRule(); + + public HBaseKerberizedConnectivityTest() { + super(miniKdcInfrastructure); + } + + @Test + public void testSqoopImportWithKerberizedHBaseConnectivitySucceeds() throws IOException { + String[] argv = getArgv(true, HBASE_TABLE_NAME, HBASE_COLUMN_FAMILY, true, null); + createTableWithColTypes(COLUMN_TYPES, new String[] { TEST_ROW_KEY, TEST_ROW_VALUE }); + + runImport(argv); + + verifyHBaseCell(HBASE_TABLE_NAME, TEST_ROW_KEY, HBASE_COLUMN_FAMILY, getColName(1), TEST_ROW_VALUE); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java index d9f7495..8b29b5f 100644 --- a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java +++ b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java @@ -18,34 +18,37 @@ package com.cloudera.sqoop.hbase; +import static org.apache.hadoop.hbase.HConstants.MASTER_INFO_PORT; +import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT; +import static org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGION_COPROCESSOR_CONF_KEY; +import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.KRB_PRINCIPAL; +import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.MASTER_KRB_PRINCIPAL; +import static org.apache.hadoop.hbase.security.User.HBASE_SECURITY_CONF_KEY; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_PRINCIPAL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import java.io.File; import java.io.IOException; -import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; -import java.util.UUID; -import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.sqoop.infrastructure.kerberos.KerberosConfigurationProvider; import org.junit.After; import org.junit.Before; @@ -58,23 +61,22 @@ import com.cloudera.sqoop.testutil.ImportJobTestCase; */ public abstract class HBaseTestCase extends ImportJobTestCase { - /* - * This is to restore test.build.data system property which gets reset - * when HBase tests are run. Since other tests in Sqoop also depend upon - * this property, they can fail if are run subsequently in the same VM. - */ - private static String testBuildDataProperty = ""; + public static final Log LOG = LogFactory.getLog( + HBaseTestCase.class.getName()); + private static final int NUM_MASTERS = 1; + private static final int NUM_SLAVES = 1; + private static final String MASTER_INFO_PORT_DISABLE_WEB_UI = "-1"; - private static void recordTestBuildDataProperty() { - testBuildDataProperty = System.getProperty("test.build.data", ""); - } + private final KerberosConfigurationProvider kerberosConfigurationProvider; + private HBaseTestingUtility hbaseTestUtil; - private static void restoreTestBuidlDataProperty() { - System.setProperty("test.build.data", testBuildDataProperty); + public HBaseTestCase() { + this(null); } - public static final Log LOG = LogFactory.getLog( - HBaseTestCase.class.getName()); + public HBaseTestCase(KerberosConfigurationProvider kerberosConfigurationProvider) { + this.kerberosConfigurationProvider = kerberosConfigurationProvider; + } /** * Create the argv to pass to Sqoop. @@ -88,8 +90,10 @@ public abstract class HBaseTestCase extends ImportJobTestCase { if (includeHadoopFlags) { CommonArgs.addHadoopFlags(args); + String zookeeperPort = hbaseTestUtil.getConfiguration().get(ZOOKEEPER_CLIENT_PORT); args.add("-D"); args.add("hbase.zookeeper.property.clientPort=" + zookeeperPort); + args.addAll(getKerberosFlags()); } if (null != queryStr) { @@ -146,76 +150,43 @@ public abstract class HBaseTestCase extends ImportJobTestCase { } return args.toArray(new String[0]); } - // Starts a mini hbase cluster in this process. - // Starts a mini hbase cluster in this process. - private HBaseTestingUtility hbaseTestUtil; - private String workDir = createTempDir().getAbsolutePath(); - private MiniZooKeeperCluster zookeeperCluster; - private MiniHBaseCluster hbaseCluster; - private int zookeeperPort; @Override @Before public void setUp() { try { - String zookeeperDir = new File(workDir, "zk").getAbsolutePath(); - zookeeperCluster = new MiniZooKeeperCluster(); - zookeeperCluster.startup(new File(zookeeperDir)); - zookeeperPort = zookeeperCluster.getClientPort(); - - HBaseTestCase.recordTestBuildDataProperty(); - String hbaseDir = new File(workDir, "hbase").getAbsolutePath(); - String hbaseRoot = "file://" + hbaseDir; - Configuration hbaseConf = HBaseConfiguration.create(); - hbaseConf.set(HConstants.HBASE_DIR, hbaseRoot); - //Hbase 0.90 does not have HConstants.ZOOKEEPER_CLIENT_PORT - hbaseConf.setInt("hbase.zookeeper.property.clientPort", zookeeperPort); - hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "0.0.0.0"); - hbaseConf.setInt("hbase.master.info.port", -1); - hbaseConf.setInt("hbase.zookeeper.property.maxClientCnxns", 500); - hbaseCluster = new MiniHBaseCluster(hbaseConf, 1); - HMaster master = hbaseCluster.getMaster(); - Object serverName = master.getServerName(); + hbaseTestUtil = new HBaseTestingUtility(); + // We set the port for the hbase master web UI to -1 because we do not want the info server to run. + hbaseTestUtil.getConfiguration().set(MASTER_INFO_PORT, MASTER_INFO_PORT_DISABLE_WEB_UI); + setupKerberos(); - String hostAndPort; - Method m; - if (serverName instanceof String) { - System.out.println("Server name is string, using HServerAddress."); - m = HMaster.class.getDeclaredMethod("getMasterAddress", - new Class<?>[]{}); - Class<?> clazz = Class.forName("org.apache.hadoop.hbase.HServerAddress"); - /* - * Call method to get server address - */ - Object serverAddr = clazz.cast(m.invoke(master, new Object[]{})); - //returns the address as hostname:port - hostAndPort = serverAddr.toString(); - } else { - System.out.println("ServerName is org.apache.hadoop.hbase.ServerName," - + "using getHostAndPort()"); - Class<?> clazz = Class.forName("org.apache.hadoop.hbase.ServerName"); - m = clazz.getDeclaredMethod("getHostAndPort", new Class<?>[]{}); - hostAndPort = m.invoke(serverName, new Object[]{}).toString(); - } - hbaseConf.set("hbase.master", hostAndPort); - hbaseTestUtil = new HBaseTestingUtility(hbaseConf); - hbaseTestUtil.setZkCluster(zookeeperCluster); - hbaseCluster.startMaster(); + hbaseTestUtil.startMiniZKCluster(); + hbaseTestUtil.startMiniHBaseCluster(NUM_MASTERS, NUM_SLAVES); super.setUp(); } catch (Throwable e) { throw new RuntimeException(e); } } + private void setupKerberos() { + if (!isKerberized()){ + return; + } + + String servicePrincipal = kerberosConfigurationProvider.getTestPrincipal() + "@" + kerberosConfigurationProvider.getRealm(); + HBaseKerberosUtils.setPrincipalForTesting(servicePrincipal); + HBaseKerberosUtils.setKeytabFileForTesting(kerberosConfigurationProvider.getKeytabFilePath()); + HBaseKerberosUtils.setSecuredConfiguration(hbaseTestUtil.getConfiguration()); + + UserGroupInformation.setConfiguration(hbaseTestUtil.getConfiguration()); + hbaseTestUtil.getConfiguration().setStrings(REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName()); + } + public void shutdown() throws Exception { LOG.info("In shutdown() method"); - if (null != hbaseTestUtil) { - LOG.info("Shutting down HBase cluster"); - hbaseCluster.shutdown(); - zookeeperCluster.shutdown(); - hbaseTestUtil = null; - } - FileUtils.deleteDirectory(new File(workDir)); + LOG.info("Shutting down HBase cluster"); + hbaseTestUtil.shutdownMiniCluster(); + hbaseTestUtil = null; LOG.info("shutdown() method returning."); } @@ -228,7 +199,6 @@ public abstract class HBaseTestCase extends ImportJobTestCase { LOG.warn("Error shutting down HBase minicluster: " + StringUtils.stringifyException(e)); } - HBaseTestCase.restoreTestBuidlDataProperty(); super.tearDown(); } @@ -252,14 +222,6 @@ public abstract class HBaseTestCase extends ImportJobTestCase { table.close(); } } - public static File createTempDir() { - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File tempDir = new File(baseDir, UUID.randomUUID().toString()); - if (tempDir.mkdir()) { - return tempDir; - } - throw new IllegalStateException("Failed to create directory"); - } protected int countHBaseTable(String tableName, String colFamily) throws IOException { @@ -278,4 +240,31 @@ public abstract class HBaseTestCase extends ImportJobTestCase { } return count; } + + protected boolean isKerberized() { + return kerberosConfigurationProvider != null; + } + + private String createFlagWithValue(String flag, String value) { + return String.format("%s=%s", flag, value); + } + + private List<String> getKerberosFlags() { + if (!isKerberized()) { + return Collections.emptyList(); + } + List<String> result = new ArrayList<>(); + + String principalForTesting = HBaseKerberosUtils.getPrincipalForTesting(); + result.add("-D"); + result.add(createFlagWithValue(HBASE_SECURITY_CONF_KEY, "kerberos")); + result.add("-D"); + result.add(createFlagWithValue(MASTER_KRB_PRINCIPAL, principalForTesting)); + result.add("-D"); + result.add(createFlagWithValue(KRB_PRINCIPAL, principalForTesting)); + result.add("-D"); + result.add(createFlagWithValue(RM_PRINCIPAL, principalForTesting)); + + return result; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java b/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java new file mode 100644 index 0000000..c149f1e --- /dev/null +++ b/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.infrastructure.kerberos; + +public interface KerberosConfigurationProvider { + + String getTestPrincipal(); + + String getRealm(); + + String getKeytabFilePath(); + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java new file mode 100644 index 0000000..5786a47 --- /dev/null +++ b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.infrastructure.kerberos; + +public interface MiniKdcInfrastructure extends KerberosConfigurationProvider { + + void start(); + + void stop(); + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java new file mode 100644 index 0000000..a704d0b --- /dev/null +++ b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.infrastructure.kerberos; + +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.minikdc.MiniKdc; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; + +public class MiniKdcInfrastructureRule implements TestRule, MiniKdcInfrastructure { + + private MiniKdc miniKdc; + + private Properties configuration; + + private File workDir; + + private String testPrincipal; + + private File keytabFile; + + public MiniKdcInfrastructureRule() { + File baseDir = Files.createTempDir(); + this.workDir = new File(baseDir, "MiniKdcWorkDir"); + this.configuration = MiniKdc.createConf(); + } + + @Override + public void start() { + try { + miniKdc = new MiniKdc(configuration, workDir); + miniKdc.start(); + createTestPrincipal(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void createTestPrincipal() { + try { + createKeytabFile(); + testPrincipal = currentUser() + "/" + miniKdc.getHost(); + miniKdc.createPrincipal(keytabFile, testPrincipal); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void createKeytabFile() { + try { + keytabFile = new File(workDir.getAbsolutePath(), "keytab"); + keytabFile.createNewFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void stop() { + try { + miniKdc.stop(); + FileUtils.deleteDirectory(workDir); + configuration = null; + miniKdc = null; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Statement apply(final Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + start(); + base.evaluate(); + stop(); + } + }; + } + + @Override + public String getTestPrincipal() { + return testPrincipal; + } + + @Override + public String getRealm() { + return miniKdc.getRealm(); + } + + @Override + public String getKeytabFilePath() { + return keytabFile.getAbsolutePath(); + } + + private String currentUser() { + return System.getProperty("user.name"); + } +}
