Repository: sqoop Updated Branches: refs/heads/trunk 010042dd4 -> bf8b8a6ac
SQOOP-3275: HBase test cases should start mini DFS cluster as well (Szabolcs Vasas via Boglarka Egyed) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/bf8b8a6a Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/bf8b8a6a Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/bf8b8a6a Branch: refs/heads/trunk Commit: bf8b8a6acd9f69c01c1bc8e42a2070f5cc596ccd Parents: 010042d Author: Boglarka Egyed <[email protected]> Authored: Mon Jan 8 18:08:37 2018 +0100 Committer: Boglarka Egyed <[email protected]> Committed: Mon Jan 8 18:08:37 2018 +0100 ---------------------------------------------------------------------- ivy.xml | 1 - .../com/cloudera/sqoop/hbase/HBaseTestCase.java | 39 ++++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/bf8b8a6a/ivy.xml ---------------------------------------------------------------------- diff --git a/ivy.xml b/ivy.xml index 86de94f..d153038 100644 --- a/ivy.xml +++ b/ivy.xml @@ -225,7 +225,6 @@ under the License. </dependency> <exclude org="org.apache.hadoop" module="avro"/> - <exclude org="commons-daemon" module="commons-daemon" /> <exclude type="pom" /> </dependencies> http://git-wip-us.apache.org/repos/asf/sqoop/blob/bf8b8a6a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java index cf42d31..99fcd70 100644 --- a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java +++ b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java @@ -24,6 +24,16 @@ import static org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGION_COPROCE import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.KRB_PRINCIPAL; import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.MASTER_KRB_PRINCIPAL; import static org.apache.hadoop.hbase.security.User.HBASE_SECURITY_CONF_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY; import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_PRINCIPAL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -49,6 +59,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.sqoop.infrastructure.kerberos.KerberosConfigurationProvider; @@ -66,9 +77,8 @@ public abstract class HBaseTestCase extends ImportJobTestCase { public static final Log LOG = LogFactory.getLog( HBaseTestCase.class.getName()); - private static final int NUM_MASTERS = 1; - private static final int NUM_SLAVES = 1; private static final String MASTER_INFO_PORT_DISABLE_WEB_UI = "-1"; + private static final String DEFAULT_DFS_HTTPS_ADDRESS = "localhost:0"; private final KerberosConfigurationProvider kerberosConfigurationProvider; private HBaseTestingUtility hbaseTestUtil; @@ -163,8 +173,7 @@ public abstract class HBaseTestCase extends ImportJobTestCase { hbaseTestUtil.getConfiguration().set(MASTER_INFO_PORT, MASTER_INFO_PORT_DISABLE_WEB_UI); setupKerberos(); - hbaseTestUtil.startMiniZKCluster(); - hbaseTestUtil.startMiniHBaseCluster(NUM_MASTERS, NUM_SLAVES); + hbaseTestUtil.startMiniCluster(); super.setUp(); } catch (Throwable e) { throw new RuntimeException(e); @@ -179,10 +188,26 @@ public abstract class HBaseTestCase extends ImportJobTestCase { String servicePrincipal = kerberosConfigurationProvider.getTestPrincipal() + "@" + kerberosConfigurationProvider.getRealm(); HBaseKerberosUtils.setPrincipalForTesting(servicePrincipal); HBaseKerberosUtils.setKeytabFileForTesting(kerberosConfigurationProvider.getKeytabFilePath()); - HBaseKerberosUtils.setSecuredConfiguration(hbaseTestUtil.getConfiguration()); - UserGroupInformation.setConfiguration(hbaseTestUtil.getConfiguration()); - hbaseTestUtil.getConfiguration().setStrings(REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName()); + Configuration configuration = hbaseTestUtil.getConfiguration(); + HBaseKerberosUtils.setSecuredConfiguration(configuration); + UserGroupInformation.setConfiguration(configuration); + configuration.setStrings(REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName()); + + setupKerberosForHdfs(servicePrincipal, configuration); + } + + private void setupKerberosForHdfs(String servicePrincipal, Configuration configuration) { + configuration.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, servicePrincipal); + configuration.set(DFS_NAMENODE_KEYTAB_FILE_KEY, kerberosConfigurationProvider.getKeytabFilePath()); + configuration.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, servicePrincipal); + configuration.set(DFS_DATANODE_KEYTAB_FILE_KEY, kerberosConfigurationProvider.getKeytabFilePath()); + configuration.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); + configuration.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, servicePrincipal); + configuration.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTP_ONLY.name()); + configuration.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, DEFAULT_DFS_HTTPS_ADDRESS); + configuration.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, DEFAULT_DFS_HTTPS_ADDRESS); + configuration.setBoolean(IGNORE_SECURE_PORTS_FOR_TESTING_KEY, true); } public void shutdown() throws Exception {
